From 6450d7f76d75303fbb5e374abf6d15bee275d6e6 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 9 Jun 2016 08:31:49 +0200 Subject: [PATCH] MongoDB metricbeat module * Contains the status metricset, reporting parts of the `db.serverStatus()` output. The locks and metrics sections are not yet reported. * Includes docs, basic integration tests, and basic system tests --- metricbeat/docker-compose.yml | 6 + metricbeat/docs/fields.asciidoc | 456 ++++++++++++++++++ metricbeat/docs/modules.asciidoc | 2 + metricbeat/docs/modules/mongodb.asciidoc | 75 +++ .../docs/modules/mongodb/status.asciidoc | 19 + metricbeat/etc/beat.full.yml | 11 + metricbeat/etc/beat.yml | 11 + metricbeat/etc/fields.yml | 338 +++++++++++++ metricbeat/include/list.go | 2 + metricbeat/metricbeat.full.yml | 11 + metricbeat/metricbeat.template-es2x.json | 295 +++++++++++ metricbeat/metricbeat.template.json | 293 +++++++++++ metricbeat/metricbeat.yml | 11 + metricbeat/module/doc.go | 4 +- metricbeat/module/mongodb/_meta/config.yml | 9 + metricbeat/module/mongodb/_meta/docs.asciidoc | 39 ++ metricbeat/module/mongodb/_meta/fields.yml | 10 + metricbeat/module/mongodb/doc.go | 4 + .../module/mongodb/status/_meta/data.json | 133 +++++ .../module/mongodb/status/_meta/docs.asciidoc | 3 + .../module/mongodb/status/_meta/fields.yml | 328 +++++++++++++ metricbeat/module/mongodb/status/data.go | 241 +++++++++ metricbeat/module/mongodb/status/status.go | 59 +++ .../mongodb/status/status_integration_test.go | 48 ++ metricbeat/module/mongodb/testing.go | 29 ++ metricbeat/tests/system/test_mongodb.py | 38 ++ 26 files changed, 2474 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/mongodb.asciidoc create mode 100644 metricbeat/docs/modules/mongodb/status.asciidoc create mode 100644 metricbeat/module/mongodb/_meta/config.yml create mode 100644 metricbeat/module/mongodb/_meta/docs.asciidoc create mode 100644 metricbeat/module/mongodb/_meta/fields.yml create mode 100644 metricbeat/module/mongodb/doc.go create mode 100644 metricbeat/module/mongodb/status/_meta/data.json create mode 100644 metricbeat/module/mongodb/status/_meta/docs.asciidoc create mode 100644 metricbeat/module/mongodb/status/_meta/fields.yml create mode 100644 metricbeat/module/mongodb/status/data.go create mode 100644 metricbeat/module/mongodb/status/status.go create mode 100644 metricbeat/module/mongodb/status/status_integration_test.go create mode 100644 metricbeat/module/mongodb/testing.go create mode 100644 metricbeat/tests/system/test_mongodb.py diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 789b9868cd6..7edc56d7303 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -2,6 +2,7 @@ beat: build: ${PWD}/. links: - apache + - mongodb - mysql - nginx - redis @@ -13,6 +14,8 @@ beat: - NGINX_PORT=80 - REDIS_HOST=redis - REDIS_PORT=6379 + - MONGODB_HOST=mongodb + - MONGODB_PORT=27017 - MYSQL_DSN=root:test@tcp(mysql:3306)/ - MYSQL_HOST=mysql - MYSQL_PORT=3306 @@ -34,6 +37,9 @@ kibana: apache: build: ${PWD}/module/apache/_meta +mongodb: + image: mongo:3.0 + mysql: image: mysql:5.7.10 environment: diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5d30c381a7b..2336a5a41c1 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -12,6 +12,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -394,6 +395,461 @@ required: True The document type. Always set to "metricsets". +[[exported-fields-mongodb]] +=== MongoDB Status Fields + +Metrics collected from MongoDB servers. + + + +[float] +=== mongodb Fields + +MongoDB metrics. + + + +[float] +=== status Fields + +MongoDB server status metrics. + + + +==== mongodb.status.version + +type: keyword + +Instance version + + +==== mongodb.status.uptime.ms + +type: long + +Instance uptime in milliseconds. + + +==== mongodb.status.local_time + +type: date + +Local time as reported by the MongoDB instance. + + +==== mongodb.status.asserts.regular + +type: long + +Number of regular assertions produced by the server. + + +==== mongodb.status.asserts.warning + +type: long + +Number of warning assertions produced by the server. + + +==== mongodb.status.asserts.msg + +type: long + +Number of msg assertions produced by the server. + + +==== mongodb.status.asserts.user + +type: long + +Number of user assertions produced by the server. + + +==== mongodb.status.asserts.rollovers + +type: long + +Number of rollovers assertions produced by the server. + + +[float] +=== background_flushing Fields + +Data about the process MongoDB uses to write data to disk. This data is only available for instances that use the MMAPv1 storage engine. + + + +==== mongodb.status.background_flushing.flushes + +type: long + +A counter that collects the number of times the database has flushed all writes to disk. + + +==== mongodb.status.background_flushing.total.ms + +type: long + +The total number of milliseconds (ms) that the mongod processes have spent writing (i.e. flushing) data to disk. Because this is an absolute value, consider the value of flushes and average_ms to provide better context for this datum. + + +==== mongodb.status.background_flushing.average.ms + +type: long + +The average time spent flushing to disk per flush event. + + +==== mongodb.status.background_flushing.last.ms + +type: long + +The amount of time, in milliseconds, that the last flush operation took to complete. + + +==== mongodb.status.background_flushing.last_finished + +type: date + +A timestamp of the last completed flush operation. + + +[float] +=== connections Fields + +Data regarding the current status of incoming connections and availability of the database server. + + + +==== mongodb.status.connections.current + +type: long + +The number of connections to the database server from clients. This number includes the current shell session. Consider the value of available to add more context to this datum. + + +==== mongodb.status.connections.available + +type: long + +The number of unused available incoming connections the database can provide. + + +==== mongodb.status.connections.total_created + +type: long + +A count of all incoming connections created to the server. This number includes connections that have since closed. + + +[float] +=== journaling Fields + +Data about the journaling-related operations and performance. Journaling information only appears for mongod instances that use the MMAPv1 storage engine and have journaling enabled. + + + +==== mongodb.status.journaling.commits + +type: long + +The number of transactions written to the journal during the last journal group commit interval. + + +==== mongodb.status.journaling.journaled.mb + +type: long + +The amount of data in megabytes (MB) written to journal during the last journal group commit interval. + + +==== mongodb.status.journaling.write_to_data_files.mb + +type: long + +The amount of data in megabytes (MB) written from journal to the data files during the last journal group commit interval. + + +==== mongodb.status.journaling.compression + +type: long + +The compression ratio of the data written to the journal. + + +==== mongodb.status.journaling.commits_in_write_lock + +type: long + +Count of the commits that occurred while a write lock was held. Commits in a write lock indicate a MongoDB node under a heavy write load and call for further diagnosis. + + +==== mongodb.status.journaling.early_commits + +type: long + +The number of times MongoDB requested a commit before the scheduled journal group commit interval. + + +[float] +=== times Fields + +Information about the performance of the mongod instance during the various phases of journaling in the last journal group commit interval. + + + +==== mongodb.status.journaling.times.dt.ms + +type: long + +The amount of time over which MongoDB collected the times data. Use this field to provide context to the other times field values. + + +==== mongodb.status.journaling.times.prep_log_buffer.ms + +type: long + +The amount of time spent preparing to write to the journal. Smaller values indicate better journal performance. + + +==== mongodb.status.journaling.times.write_to_journal.ms + +type: long + +The amount of time spent actually writing to the journal. File system speeds and device interfaces can affect performance. + + +==== mongodb.status.journaling.times.write_to_data_files.ms + +type: long + +The amount of time spent writing to data files after journaling. File system speeds and device interfaces can affect performance. + + +==== mongodb.status.journaling.times.remap_private_view.ms + +type: long + +The amount of time spent remapping copy-on-write memory mapped views. Smaller values indicate better journal performance. + + +==== mongodb.status.journaling.times.commits.ms + +type: long + +The amount of time spent for commits. + + +==== mongodb.status.journaling.times.commits_in_write_lock.ms + +type: long + +The amount of time spent for commits that occurred while a write lock was held. + + +[float] +=== extra_info Fields + +Platform specific data. + + + +==== mongodb.status.extra_info.heap_usage.bytes + +type: long + +The total size in bytes of heap space used by the database process. Only available on Unix/Linux. + + +==== mongodb.status.extra_info.page_faults + +type: long + +The total number of page faults that require disk operations. Page faults refer to operations that require the database server to access data which isn't available in active memory. + + +[float] +=== network Fields + +Platform specific data. + + + +==== mongodb.status.network.in.bytes + +type: long + +The amount of network traffic, in bytes, received by this database. + + +==== mongodb.status.network.out.bytes + +type: long + +The amount of network traffic, in bytes, sent from this database. + + +==== mongodb.status.network.requests + +type: long + +The amount of network traffic, in bytes, sent from this database. + + +[float] +=== opcounters Fields + +An overview of database operations by type. + + + +==== mongodb.status.opcounters.insert + +type: long + +The total number of insert operations received since the mongod instance last started. + + +==== mongodb.status.opcounters.query + +type: long + +The total number of queries received since the mongod instance last started. + + +==== mongodb.status.opcounters.update + +type: long + +The total number of update operations received since the mongod instance last started. + + +==== mongodb.status.opcounters.delete + +type: long + +The total number of delete operations received since the mongod instance last started. + + +==== mongodb.status.opcounters.getmore + +type: long + +The total number of getmore operations received since the mongod instance last started. + + +==== mongodb.status.opcounters.command + +type: long + +The total number of commands issued to the database since the mongod instance last started. + + +[float] +=== opcounters_replicated Fields + +An overview of database replication operations by type. + + + +==== mongodb.status.opcounters_replicated.insert + +type: long + +The total number of replicated insert operations received since the mongod instance last started. + + +==== mongodb.status.opcounters_replicated.query + +type: long + +The total number of replicated queries received since the mongod instance last started. + + +==== mongodb.status.opcounters_replicated.update + +type: long + +The total number of replicated update operations received since the mongod instance last started. + + +==== mongodb.status.opcounters_replicated.delete + +type: long + +The total number of replicated delete operations received since the mongod instance last started. + + +==== mongodb.status.opcounters_replicated.getmore + +type: long + +The total number of replicated getmore operations received since the mongod instance last started. + + +==== mongodb.status.opcounters_replicated.command + +type: long + +The total number of replicated commands issued to the database since the mongod instance last started. + + +[float] +=== memory Fields + +Data about the current memory usage of the mongod server. + + + +==== mongodb.status.memory.bits + +type: long + +Either 64 or 32, depending on which target architecture specified during the mongod compilation process. + + +==== mongodb.status.memory.resident.mb + +type: long + +The amount of RAM, in megabytes (MB), currently used by the database process. + + +==== mongodb.status.memory.virtual.mb + +type: long + +The amount, in megabytes (MB), of virtual memory used by the mongod process. + + +==== mongodb.status.memory.mapped.mb + +type: long + +The amount of mapped memory, in megabytes (MB), by the database. Because MongoDB uses memory-mapped files, this value is likely to be to be roughly equivalent to the total size of your database or databases. + + +==== mongodb.status.memory.mapped_with_journal.mb + +type: long + +The amount of mapped memory, in megabytes (MB), including the memory used for journaling. + + +==== mongodb.status.write_backs_queued + +type: boolean + +True when there are operations from a mongos instance queued for retrying. + + +==== mongodb.status.storage_engine.name + +type: keyword + +A string that represents the name of the current storage engine. + + [[exported-fields-mysql]] === MySQL Status Fields diff --git a/metricbeat/docs/modules.asciidoc b/metricbeat/docs/modules.asciidoc index 8de99357f32..221e26510f5 100644 --- a/metricbeat/docs/modules.asciidoc +++ b/metricbeat/docs/modules.asciidoc @@ -8,6 +8,7 @@ contained in {beatname_uc}. Each module contains one or multiple metricsets. Mor about each module can be found under the links below. * <> + * <> * <> * <> * <> @@ -17,6 +18,7 @@ about each module can be found under the links below. -- include::modules/apache.asciidoc[] +include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] include::modules/redis.asciidoc[] diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc new file mode 100644 index 00000000000..6517679aed3 --- /dev/null +++ b/metricbeat/docs/modules/mongodb.asciidoc @@ -0,0 +1,75 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-mongodb]] +== MongoDB Module + +This modules periodically fetches metrics from https://www.mongodb.com[MongoDB] +servers. + +[float] +=== Configuration Options + +A basic configuration sample for the MongoDB module can be found in the next section. + +==== hosts + +The hosts must be passed as a MongoDB URLs in the following format: + +----------------------------------- +[mongodb://][user:pass@]host[:port] +----------------------------------- + +It can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["mongodb://myuser:mypass@localhost:40001", "otherhost:40001"] +---------------------------------------------------------------------- + + +[float] +=== Compatibility + +The mongodb MetricSets were tested with MongoDB 3.0 are expected to work with all versions >= 2.8. + + +[float] +=== Example Configuration + +The MongoDB Status module supports the standard configuration options which can be found +here <>. Below is an example of a configuration option: + +[source,yaml] +---- +metricbeat.modules: +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + +---- + +[float] +=== MetricSets + +The following MetricSets are available: + +* <> + +include::mongodb/status.asciidoc[] + diff --git a/metricbeat/docs/modules/mongodb/status.asciidoc b/metricbeat/docs/modules/mongodb/status.asciidoc new file mode 100644 index 00000000000..552560fe8c7 --- /dev/null +++ b/metricbeat/docs/modules/mongodb/status.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-mongodb-status]] +include::../../../module/mongodb/status/_meta/docs.asciidoc[] + + +==== Fields + +A description of each field in the MetricSet can be found in the +<> section + +Below is an example document generated by this metricset. + +[source,json] +---- +include::../../../module/mongodb/status/_meta/data.json[] +---- diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml index e55edbb2852..c7b0b694b15 100644 --- a/metricbeat/etc/beat.full.yml +++ b/metricbeat/etc/beat.full.yml @@ -61,6 +61,17 @@ metricbeat.modules: # Password of hosts. Empty by default #password: test123 +#--------------------------- MongoDB Status Module --------------------------- +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + + #---------------------------- MySQL Status Module ---------------------------- #- module: mysql #metricsets: ["status"] diff --git a/metricbeat/etc/beat.yml b/metricbeat/etc/beat.yml index db1378033ea..8bcce46ffd9 100644 --- a/metricbeat/etc/beat.yml +++ b/metricbeat/etc/beat.yml @@ -43,4 +43,15 @@ metricbeat.modules: # if true, exports the CPU usage in ticks, together with the percentage values cpu_ticks: false +#--------------------------- MongoDB Status Module --------------------------- +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + + diff --git a/metricbeat/etc/fields.yml b/metricbeat/etc/fields.yml index dc7188b8af6..61764878bb0 100644 --- a/metricbeat/etc/fields.yml +++ b/metricbeat/etc/fields.yml @@ -205,6 +205,344 @@ type: integer description: > Total. +- key: mongodb + title: "MongoDB Status" + description: > + Metrics collected from MongoDB servers. + fields: + - name: mongodb + type: group + description: > + MongoDB metrics. + fields: + - name: status + type: group + description: > + MongoDB server status metrics. + fields: + - name: version + type: keyword + description: > + Instance version + - name: uptime.ms + type: long + description: > + Instance uptime in milliseconds. + - name: local_time + type: date + description: > + Local time as reported by the MongoDB instance. + + - name: asserts.regular + type: long + description: > + Number of regular assertions produced by the server. + - name: asserts.warning + type: long + description: > + Number of warning assertions produced by the server. + - name: asserts.msg + type: long + description: > + Number of msg assertions produced by the server. + - name: asserts.user + type: long + description: > + Number of user assertions produced by the server. + - name: asserts.rollovers + type: long + description: > + Number of rollovers assertions produced by the server. + + - name: background_flushing + type: group + description: > + Data about the process MongoDB uses to write data to disk. This data is + only available for instances that use the MMAPv1 storage engine. + fields: + - name: flushes + type: long + description: > + A counter that collects the number of times the database has + flushed all writes to disk. + - name: total.ms + type: long + description: > + The total number of milliseconds (ms) that the mongod processes have + spent writing (i.e. flushing) data to disk. Because this is an + absolute value, consider the value of flushes and average_ms to + provide better context for this datum. + - name: average.ms + type: long + description: > + The average time spent flushing to disk per flush event. + - name: last.ms + type: long + description: > + The amount of time, in milliseconds, that the last flush operation + took to complete. + - name: last_finished + type: date + description: > + A timestamp of the last completed flush operation. + + - name: connections + type: group + description: > + Data regarding the current status of incoming connections and + availability of the database server. + fields: + - name: current + type: long + description: > + The number of connections to the database server from clients. This + number includes the current shell session. Consider the value of + available to add more context to this datum. + - name: available + type: long + description: > + The number of unused available incoming connections the database + can provide. + - name: total_created + type: long + description: > + A count of all incoming connections created to the server. This + number includes connections that have since closed. + + - name: journaling + type: group + description: > + Data about the journaling-related operations and performance. Journaling + information only appears for mongod instances that use the MMAPv1 + storage engine and have journaling enabled. + fields: + - name: commits + type: long + description: > + The number of transactions written to the journal during the last + journal group commit interval. + - name: journaled.mb + type: long + description: > + The amount of data in megabytes (MB) written to journal during the + last journal group commit interval. + - name: write_to_data_files.mb + type: long + description: > + The amount of data in megabytes (MB) written from journal to the + data files during the last journal group commit interval. + - name: compression + type: long + description: > + The compression ratio of the data written to the journal. + - name: commits_in_write_lock + type: long + description: > + Count of the commits that occurred while a write lock was held. + Commits in a write lock indicate a MongoDB node under a heavy write + load and call for further diagnosis. + - name: early_commits + type: long + description: > + The number of times MongoDB requested a commit before the scheduled + journal group commit interval. + - name: times + type: group + description: > + Information about the performance of the mongod instance during the + various phases of journaling in the last journal group commit + interval. + fields: + - name: dt.ms + type: long + description: > + The amount of time over which MongoDB collected the times data. + Use this field to provide context to the other times field values. + - name: prep_log_buffer.ms + type: long + description: > + The amount of time spent preparing to write to the journal. + Smaller values indicate better journal performance. + - name: write_to_journal.ms + type: long + description: > + The amount of time spent actually writing to the journal. File + system speeds and device interfaces can affect performance. + - name: write_to_data_files.ms + type: long + description: > + The amount of time spent writing to data files after journaling. + File system speeds and device interfaces can affect performance. + - name: remap_private_view.ms + type: long + description: > + The amount of time spent remapping copy-on-write memory mapped + views. Smaller values indicate better journal performance. + - name: commits.ms + type: long + description: > + The amount of time spent for commits. + - name: commits_in_write_lock.ms + type: long + description: > + The amount of time spent for commits that occurred while a write + lock was held. + + - name: extra_info + type: group + description: > + Platform specific data. + fields: + - name: heap_usage.bytes + type: long + description: > + The total size in bytes of heap space used by the database process. + Only available on Unix/Linux. + - name: page_faults + type: long + description: > + The total number of page faults that require disk operations. Page + faults refer to operations that require the database server to + access data which isn't available in active memory. + + - name: network + type: group + description: > + Platform specific data. + fields: + - name: in.bytes + type: long + description: > + The amount of network traffic, in bytes, received by this database. + - name: out.bytes + type: long + description: > + The amount of network traffic, in bytes, sent from this database. + - name: requests + type: long + description: > + The amount of network traffic, in bytes, sent from this database. + + - name: opcounters + type: group + description: > + An overview of database operations by type. + fields: + - name: insert + type: long + description: > + The total number of insert operations received since the mongod + instance last started. + - name: query + type: long + description: > + The total number of queries received since the mongod instance last + started. + - name: update + type: long + description: > + The total number of update operations received since the mongod + instance last started. + - name: delete + type: long + description: > + The total number of delete operations received since the mongod + instance last started. + - name: getmore + type: long + description: > + The total number of getmore operations received since the mongod + instance last started. + - name: command + type: long + description: > + The total number of commands issued to the database since the mongod + instance last started. + + - name: opcounters_replicated + type: group + description: > + An overview of database replication operations by type. + fields: + - name: insert + type: long + description: > + The total number of replicated insert operations received since the + mongod instance last started. + - name: query + type: long + description: > + The total number of replicated queries received since the mongod + instance last started. + - name: update + type: long + description: > + The total number of replicated update operations received since the + mongod instance last started. + - name: delete + type: long + description: > + The total number of replicated delete operations received since the + mongod instance last started. + - name: getmore + type: long + description: > + The total number of replicated getmore operations received since the + mongod instance last started. + - name: command + type: long + description: > + The total number of replicated commands issued to the database since + the mongod instance last started. + + - name: memory + type: group + description: > + Data about the current memory usage of the mongod server. + fields: + - name: bits + type: long + description: > + Either 64 or 32, depending on which target architecture specified + during the mongod compilation process. + - name: resident.mb + type: long + description: > + The amount of RAM, in megabytes (MB), currently used by the database + process. + - name: virtual.mb + type: long + description: > + The amount, in megabytes (MB), of virtual memory used by the mongod + process. + - name: mapped.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), by the database. + Because MongoDB uses memory-mapped files, this value is likely to be + to be roughly equivalent to the total size of your database or + databases. + - name: mapped_with_journal.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), including the memory + used for journaling. + - name: write_backs_queued + type: boolean + description: > + True when there are operations from a mongos instance queued for retrying. + - name: storage_engine.name + type: keyword + description: > + A string that represents the name of the current storage engine. + + + + + + + + - key: mysql title: "MySQL Status" description: > diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 69a7e99db89..d785586d17e 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -10,6 +10,8 @@ import ( // This list is automatically generated by `make imports` _ "github.com/elastic/beats/metricbeat/module/apache" _ "github.com/elastic/beats/metricbeat/module/apache/status" + _ "github.com/elastic/beats/metricbeat/module/mongodb" + _ "github.com/elastic/beats/metricbeat/module/mongodb/status" _ "github.com/elastic/beats/metricbeat/module/mysql" _ "github.com/elastic/beats/metricbeat/module/mysql/status" _ "github.com/elastic/beats/metricbeat/module/nginx" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index f6bce4745e9..c012a99e34f 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -61,6 +61,17 @@ metricbeat.modules: # Password of hosts. Empty by default #password: test123 +#--------------------------- MongoDB Status Module --------------------------- +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + + #---------------------------- MySQL Status Module ---------------------------- #- module: mysql #metricsets: ["status"] diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 9370b018f8c..49284ee9841 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -199,6 +199,301 @@ } } }, + "mongodb": { + "properties": { + "status": { + "properties": { + "asserts": { + "properties": { + "msg": { + "type": "long" + }, + "regular": { + "type": "long" + }, + "rollovers": { + "type": "long" + }, + "user": { + "type": "long" + }, + "warning": { + "type": "long" + } + } + }, + "background_flushing": { + "properties": { + "average": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "flushes": { + "type": "long" + }, + "last": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "last_finished": { + "type": "date" + }, + "total": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "connections": { + "properties": { + "available": { + "type": "long" + }, + "current": { + "type": "long" + }, + "total_created": { + "type": "long" + } + } + }, + "extra_info": { + "properties": { + "heap_usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + } + } + }, + "journaling": { + "properties": { + "commits": { + "type": "long" + }, + "commits_in_write_lock": { + "type": "long" + }, + "compression": { + "type": "long" + }, + "early_commits": { + "type": "long" + }, + "journaled": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "times": { + "properties": { + "commits": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "commits_in_write_lock": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "dt": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "prep_log_buffer": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "remap_private_view": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_data_files": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_journal": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "write_to_data_files": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "local_time": { + "type": "date" + }, + "memory": { + "properties": { + "bits": { + "type": "long" + }, + "mapped": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "mapped_with_journal": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "resident": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "requests": { + "type": "long" + } + } + }, + "opcounters": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "opcounters_replicated": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "storage_engine": { + "properties": { + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "write_backs_queued": { + "type": "boolean" + } + } + } + } + }, "mysql": { "properties": { "status": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 8b129aa8142..cfaf9e05b0c 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -190,6 +190,299 @@ } } }, + "mongodb": { + "properties": { + "status": { + "properties": { + "asserts": { + "properties": { + "msg": { + "type": "long" + }, + "regular": { + "type": "long" + }, + "rollovers": { + "type": "long" + }, + "user": { + "type": "long" + }, + "warning": { + "type": "long" + } + } + }, + "background_flushing": { + "properties": { + "average": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "flushes": { + "type": "long" + }, + "last": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "last_finished": { + "type": "date" + }, + "total": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "connections": { + "properties": { + "available": { + "type": "long" + }, + "current": { + "type": "long" + }, + "total_created": { + "type": "long" + } + } + }, + "extra_info": { + "properties": { + "heap_usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + } + } + }, + "journaling": { + "properties": { + "commits": { + "type": "long" + }, + "commits_in_write_lock": { + "type": "long" + }, + "compression": { + "type": "long" + }, + "early_commits": { + "type": "long" + }, + "journaled": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "times": { + "properties": { + "commits": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "commits_in_write_lock": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "dt": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "prep_log_buffer": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "remap_private_view": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_data_files": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_journal": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "write_to_data_files": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "local_time": { + "type": "date" + }, + "memory": { + "properties": { + "bits": { + "type": "long" + }, + "mapped": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "mapped_with_journal": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "resident": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "requests": { + "type": "long" + } + } + }, + "opcounters": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "opcounters_replicated": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "storage_engine": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "write_backs_queued": { + "type": "boolean" + } + } + } + } + }, "mysql": { "properties": { "status": { diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 9e2bc326a94..9069a76d823 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -43,6 +43,17 @@ metricbeat.modules: # if true, exports the CPU usage in ticks, together with the percentage values cpu_ticks: false +#--------------------------- MongoDB Status Module --------------------------- +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + + #================================ General ===================================== diff --git a/metricbeat/module/doc.go b/metricbeat/module/doc.go index 20166704e73..75cca56c481 100644 --- a/metricbeat/module/doc.go +++ b/metricbeat/module/doc.go @@ -34,7 +34,9 @@ List of standardised words and units across all metricsets. On the left are the * pct: precentage * request: req * sec: seconds, second, s -* +* ms: millisecond, millis +* mb: megabytes +* msg: message */ package module diff --git a/metricbeat/module/mongodb/_meta/config.yml b/metricbeat/module/mongodb/_meta/config.yml new file mode 100644 index 00000000000..660b683f6d3 --- /dev/null +++ b/metricbeat/module/mongodb/_meta/config.yml @@ -0,0 +1,9 @@ +- module: mongodb + metricsets: ["status"] + enabled: true + period: 10s + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port] + hosts: ["localhost:27017"] + diff --git a/metricbeat/module/mongodb/_meta/docs.asciidoc b/metricbeat/module/mongodb/_meta/docs.asciidoc new file mode 100644 index 00000000000..f5b3deca9d9 --- /dev/null +++ b/metricbeat/module/mongodb/_meta/docs.asciidoc @@ -0,0 +1,39 @@ +== MongoDB Module + +This modules periodically fetches metrics from https://www.mongodb.com[MongoDB] +servers. + +[float] +=== Configuration Options + +A basic configuration sample for the MongoDB module can be found in the next section. + +==== hosts + +The hosts must be passed as a MongoDB URLs in the following format: + +----------------------------------- +[mongodb://][user:pass@]host[:port] +----------------------------------- + +It can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: mongodb + hosts: ["mongodb://myuser:mypass@localhost:40001", "otherhost:40001"] +---------------------------------------------------------------------- + + +[float] +=== Compatibility + +The mongodb MetricSets were tested with MongoDB 3.0 are expected to work with all versions >= 2.8. diff --git a/metricbeat/module/mongodb/_meta/fields.yml b/metricbeat/module/mongodb/_meta/fields.yml new file mode 100644 index 00000000000..2897bdadca4 --- /dev/null +++ b/metricbeat/module/mongodb/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: mongodb + title: "MongoDB Status" + description: > + Metrics collected from MongoDB servers. + fields: + - name: mongodb + type: group + description: > + MongoDB metrics. + fields: diff --git a/metricbeat/module/mongodb/doc.go b/metricbeat/module/mongodb/doc.go new file mode 100644 index 00000000000..c43ed399329 --- /dev/null +++ b/metricbeat/module/mongodb/doc.go @@ -0,0 +1,4 @@ +/* +Package mongodb is a Metricbeat module that contains MetricSets. +*/ +package mongodb diff --git a/metricbeat/module/mongodb/status/_meta/data.json b/metricbeat/module/mongodb/status/_meta/data.json new file mode 100644 index 00000000000..59398fadcce --- /dev/null +++ b/metricbeat/module/mongodb/status/_meta/data.json @@ -0,0 +1,133 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost", + "module": "mongodb", + "name": "status", + "rtt": 115 + }, + "mongodb": { + "status": { + "asserts": { + "msg": 0, + "regular": 0, + "rollovers": 0, + "user": 0, + "warning": 0 + }, + "background_flushing": { + "average": { + "ms": 2 + }, + "flushes": 68, + "last": { + "ms": 2 + }, + "last_finished": "2016-06-13T07:59:58.867Z", + "total": { + "ms": 145 + } + }, + "connections": { + "available": 838859, + "current": 1, + "total_created": 13 + }, + "extra_info": { + "heap_usage": { + "bytes": 62893688 + }, + "page_faults": 127 + }, + "journaling": { + "commits": 25, + "commits_in_write_lock": 0, + "compression": 0, + "early_commits": 0, + "journaled": { + "mb": 0 + }, + "times": { + "commits": { + "ms": 0 + }, + "commits_in_write_lock": { + "ms": 0 + }, + "dt": { + "ms": 0 + }, + "prep_log_buffer": { + "ms": 0 + }, + "remap_private_view": { + "ms": 0 + }, + "write_to_data_files": { + "ms": 0 + }, + "write_to_journal": { + "ms": 0 + } + }, + "write_to_data_files": { + "mb": 0 + } + }, + "local_time": "2016-06-13T08:00:03.509Z", + "memory": { + "bits": 64, + "mapped": { + "mb": 80 + }, + "mapped_with_journal": { + "mb": 160 + }, + "resident": { + "mb": 42 + }, + "virtual": { + "mb": 356 + } + }, + "network": { + "in": { + "bytes": 3656 + }, + "out": { + "bytes": 124556 + }, + "requests": 64 + }, + "opcounters": { + "command": 65, + "delete": 0, + "getmore": 0, + "insert": 0, + "query": 1, + "update": 0 + }, + "opcounters_replicated": { + "command": 0, + "delete": 0, + "getmore": 0, + "insert": 0, + "query": 0, + "update": 0 + }, + "storage_engine": { + "name": "mmapv1" + }, + "uptime": { + "ms": 37201270 + }, + "version": "3.0.12", + "write_backs_queued": false + } + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/mongodb/status/_meta/docs.asciidoc b/metricbeat/module/mongodb/status/_meta/docs.asciidoc new file mode 100644 index 00000000000..6db4b3d5cf4 --- /dev/null +++ b/metricbeat/module/mongodb/status/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== mongodb status MetricSet + +This is the status Metricset of the module mongodb. diff --git a/metricbeat/module/mongodb/status/_meta/fields.yml b/metricbeat/module/mongodb/status/_meta/fields.yml new file mode 100644 index 00000000000..2d8628aff60 --- /dev/null +++ b/metricbeat/module/mongodb/status/_meta/fields.yml @@ -0,0 +1,328 @@ +- name: status + type: group + description: > + MongoDB server status metrics. + fields: + - name: version + type: keyword + description: > + Instance version + - name: uptime.ms + type: long + description: > + Instance uptime in milliseconds. + - name: local_time + type: date + description: > + Local time as reported by the MongoDB instance. + + - name: asserts.regular + type: long + description: > + Number of regular assertions produced by the server. + - name: asserts.warning + type: long + description: > + Number of warning assertions produced by the server. + - name: asserts.msg + type: long + description: > + Number of msg assertions produced by the server. + - name: asserts.user + type: long + description: > + Number of user assertions produced by the server. + - name: asserts.rollovers + type: long + description: > + Number of rollovers assertions produced by the server. + + - name: background_flushing + type: group + description: > + Data about the process MongoDB uses to write data to disk. This data is + only available for instances that use the MMAPv1 storage engine. + fields: + - name: flushes + type: long + description: > + A counter that collects the number of times the database has + flushed all writes to disk. + - name: total.ms + type: long + description: > + The total number of milliseconds (ms) that the mongod processes have + spent writing (i.e. flushing) data to disk. Because this is an + absolute value, consider the value of flushes and average_ms to + provide better context for this datum. + - name: average.ms + type: long + description: > + The average time spent flushing to disk per flush event. + - name: last.ms + type: long + description: > + The amount of time, in milliseconds, that the last flush operation + took to complete. + - name: last_finished + type: date + description: > + A timestamp of the last completed flush operation. + + - name: connections + type: group + description: > + Data regarding the current status of incoming connections and + availability of the database server. + fields: + - name: current + type: long + description: > + The number of connections to the database server from clients. This + number includes the current shell session. Consider the value of + available to add more context to this datum. + - name: available + type: long + description: > + The number of unused available incoming connections the database + can provide. + - name: total_created + type: long + description: > + A count of all incoming connections created to the server. This + number includes connections that have since closed. + + - name: journaling + type: group + description: > + Data about the journaling-related operations and performance. Journaling + information only appears for mongod instances that use the MMAPv1 + storage engine and have journaling enabled. + fields: + - name: commits + type: long + description: > + The number of transactions written to the journal during the last + journal group commit interval. + - name: journaled.mb + type: long + description: > + The amount of data in megabytes (MB) written to journal during the + last journal group commit interval. + - name: write_to_data_files.mb + type: long + description: > + The amount of data in megabytes (MB) written from journal to the + data files during the last journal group commit interval. + - name: compression + type: long + description: > + The compression ratio of the data written to the journal. + - name: commits_in_write_lock + type: long + description: > + Count of the commits that occurred while a write lock was held. + Commits in a write lock indicate a MongoDB node under a heavy write + load and call for further diagnosis. + - name: early_commits + type: long + description: > + The number of times MongoDB requested a commit before the scheduled + journal group commit interval. + - name: times + type: group + description: > + Information about the performance of the mongod instance during the + various phases of journaling in the last journal group commit + interval. + fields: + - name: dt.ms + type: long + description: > + The amount of time over which MongoDB collected the times data. + Use this field to provide context to the other times field values. + - name: prep_log_buffer.ms + type: long + description: > + The amount of time spent preparing to write to the journal. + Smaller values indicate better journal performance. + - name: write_to_journal.ms + type: long + description: > + The amount of time spent actually writing to the journal. File + system speeds and device interfaces can affect performance. + - name: write_to_data_files.ms + type: long + description: > + The amount of time spent writing to data files after journaling. + File system speeds and device interfaces can affect performance. + - name: remap_private_view.ms + type: long + description: > + The amount of time spent remapping copy-on-write memory mapped + views. Smaller values indicate better journal performance. + - name: commits.ms + type: long + description: > + The amount of time spent for commits. + - name: commits_in_write_lock.ms + type: long + description: > + The amount of time spent for commits that occurred while a write + lock was held. + + - name: extra_info + type: group + description: > + Platform specific data. + fields: + - name: heap_usage.bytes + type: long + description: > + The total size in bytes of heap space used by the database process. + Only available on Unix/Linux. + - name: page_faults + type: long + description: > + The total number of page faults that require disk operations. Page + faults refer to operations that require the database server to + access data which isn't available in active memory. + + - name: network + type: group + description: > + Platform specific data. + fields: + - name: in.bytes + type: long + description: > + The amount of network traffic, in bytes, received by this database. + - name: out.bytes + type: long + description: > + The amount of network traffic, in bytes, sent from this database. + - name: requests + type: long + description: > + The amount of network traffic, in bytes, sent from this database. + + - name: opcounters + type: group + description: > + An overview of database operations by type. + fields: + - name: insert + type: long + description: > + The total number of insert operations received since the mongod + instance last started. + - name: query + type: long + description: > + The total number of queries received since the mongod instance last + started. + - name: update + type: long + description: > + The total number of update operations received since the mongod + instance last started. + - name: delete + type: long + description: > + The total number of delete operations received since the mongod + instance last started. + - name: getmore + type: long + description: > + The total number of getmore operations received since the mongod + instance last started. + - name: command + type: long + description: > + The total number of commands issued to the database since the mongod + instance last started. + + - name: opcounters_replicated + type: group + description: > + An overview of database replication operations by type. + fields: + - name: insert + type: long + description: > + The total number of replicated insert operations received since the + mongod instance last started. + - name: query + type: long + description: > + The total number of replicated queries received since the mongod + instance last started. + - name: update + type: long + description: > + The total number of replicated update operations received since the + mongod instance last started. + - name: delete + type: long + description: > + The total number of replicated delete operations received since the + mongod instance last started. + - name: getmore + type: long + description: > + The total number of replicated getmore operations received since the + mongod instance last started. + - name: command + type: long + description: > + The total number of replicated commands issued to the database since + the mongod instance last started. + + - name: memory + type: group + description: > + Data about the current memory usage of the mongod server. + fields: + - name: bits + type: long + description: > + Either 64 or 32, depending on which target architecture specified + during the mongod compilation process. + - name: resident.mb + type: long + description: > + The amount of RAM, in megabytes (MB), currently used by the database + process. + - name: virtual.mb + type: long + description: > + The amount, in megabytes (MB), of virtual memory used by the mongod + process. + - name: mapped.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), by the database. + Because MongoDB uses memory-mapped files, this value is likely to be + to be roughly equivalent to the total size of your database or + databases. + - name: mapped_with_journal.mb + type: long + description: > + The amount of mapped memory, in megabytes (MB), including the memory + used for journaling. + - name: write_backs_queued + type: boolean + description: > + True when there are operations from a mongos instance queued for retrying. + - name: storage_engine.name + type: keyword + description: > + A string that represents the name of the current storage engine. + + + + + + + + diff --git a/metricbeat/module/mongodb/status/data.go b/metricbeat/module/mongodb/status/data.go new file mode 100644 index 00000000000..e7fe6f2465e --- /dev/null +++ b/metricbeat/module/mongodb/status/data.go @@ -0,0 +1,241 @@ +package status + +import ( + "fmt" + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +func eventMapping(status map[string]interface{}) common.MapStr { + errs := map[string]error{} + event := common.MapStr{ + "version": mustBeString("version", status, errs), + "uptime": common.MapStr{ + "ms": mustBeInteger("uptimeMillis", status, errs), + }, + "local_time": mustBeTime("localTime", status, errs), + "write_backs_queued": mustBeBool("writeBacksQueued", status, errs), + } + removeErroredKeys(event, errs) + + asserts, ok := status["asserts"].(map[string]interface{}) + if ok { + errs := map[string]error{} + asserts := common.MapStr{ + "regular": mustBeInteger("regular", asserts, errs), + "warning": mustBeInteger("warning", asserts, errs), + "msg": mustBeInteger("msg", asserts, errs), + "user": mustBeInteger("user", asserts, errs), + "rollovers": mustBeInteger("rollovers", asserts, errs), + } + removeErroredKeys(asserts, errs) + event["asserts"] = asserts + } + + backgroundFlushing, ok := status["backgroundFlushing"].(map[string]interface{}) + if ok { + errs := map[string]error{} + backgroundFlushing = common.MapStr{ + "flushes": mustBeInteger("flushes", backgroundFlushing, errs), + "total": common.MapStr{ + "ms": mustBeInteger("total_ms", backgroundFlushing, errs), + }, + "average": common.MapStr{ + "ms": mustBeInteger("average_ms", backgroundFlushing, errs), + }, + "last": common.MapStr{ + "ms": mustBeInteger("last_ms", backgroundFlushing, errs), + }, + "last_finished": mustBeTime("last_finished", backgroundFlushing, errs), + } + removeErroredKeys(backgroundFlushing, errs) + event["background_flushing"] = backgroundFlushing + } + + connections, ok := status["connections"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventConnections := common.MapStr{ + "current": mustBeInteger("current", connections, errs), + "available": mustBeInteger("available", connections, errs), + "total_created": mustBeInteger("totalCreated", connections, errs), + } + removeErroredKeys(eventConnections, errs) + event["connections"] = eventConnections + } + + dur, ok := status["dur"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventDur := common.MapStr{ + "commits": mustBeInteger("commits", dur, errs), + "journaled": common.MapStr{ + "mb": mustBeInteger("journaledMB", dur, errs), + }, + "write_to_data_files": common.MapStr{ + "mb": mustBeInteger("writeToDataFilesMB", dur, errs), + }, + "compression": mustBeInteger("compression", dur, errs), + "commits_in_write_lock": mustBeInteger("commitsInWriteLock", dur, errs), + "early_commits": mustBeInteger("earlyCommits", dur, errs), + } + times, ok := dur["timeMs"].(map[string]interface{}) + if ok { + errs := map[string]error{} + times = common.MapStr{ + "dt": common.MapStr{"ms": mustBeInteger("dt", times, errs)}, + "prep_log_buffer": common.MapStr{"ms": mustBeInteger("prepLogBuffer", times, errs)}, + "write_to_journal": common.MapStr{"ms": mustBeInteger("writeToJournal", times, errs)}, + "write_to_data_files": common.MapStr{"ms": mustBeInteger("writeToDataFiles", times, errs)}, + "remap_private_view": common.MapStr{"ms": mustBeInteger("remapPrivateView", times, errs)}, + "commits": common.MapStr{"ms": mustBeInteger("commits", times, errs)}, + "commits_in_write_lock": common.MapStr{"ms": mustBeInteger("commitsInWriteLock", times, errs)}, + } + removeErroredKeys(times, errs) + eventDur["times"] = times + } + removeErroredKeys(eventDur, errs) + event["journaling"] = eventDur + } + + extraInfo, ok := status["extra_info"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventExtraInfo := common.MapStr{ + "heap_usage": common.MapStr{"bytes": mustBeInteger("heap_usage_bytes", extraInfo, errs)}, + "page_faults": mustBeInteger("page_faults", extraInfo, errs), + } + removeErroredKeys(eventExtraInfo, errs) + event["extra_info"] = eventExtraInfo + } + + network, ok := status["network"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventNetwork := common.MapStr{ + "in": common.MapStr{"bytes": mustBeInteger("bytesIn", network, errs)}, + "out": common.MapStr{"bytes": mustBeInteger("bytesOut", network, errs)}, + "requests": mustBeInteger("numRequests", network, errs), + } + removeErroredKeys(eventNetwork, errs) + event["network"] = eventNetwork + } + + opcounters, ok := status["opcounters"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventOpcounters := common.MapStr{} + for key, _ := range opcounters { + eventOpcounters[key] = mustBeInteger(key, opcounters, errs) + } + removeErroredKeys(eventOpcounters, errs) + event["opcounters"] = eventOpcounters + } + + opcountersRepl, ok := status["opcountersRepl"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventOpcountersRepl := common.MapStr{} + for key, _ := range opcountersRepl { + eventOpcountersRepl[key] = mustBeInteger(key, opcountersRepl, errs) + } + removeErroredKeys(eventOpcountersRepl, errs) + event["opcounters_replicated"] = eventOpcountersRepl + } + + mem, ok := status["mem"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventMem := common.MapStr{ + "bits": mustBeInteger("bits", mem, errs), + "resident": common.MapStr{"mb": mustBeInteger("resident", mem, errs)}, + "virtual": common.MapStr{"mb": mustBeInteger("virtual", mem, errs)}, + "mapped": common.MapStr{"mb": mustBeInteger("mapped", mem, errs)}, + "mapped_with_journal": common.MapStr{"mb": mustBeInteger("mappedWithJournal", mem, errs)}, + } + removeErroredKeys(eventMem, errs) + event["memory"] = eventMem + } + + storageEngine, ok := status["storageEngine"].(map[string]interface{}) + if ok { + errs := map[string]error{} + eventStorageEngine := common.MapStr{ + "name": mustBeString("name", storageEngine, errs), + } + removeErroredKeys(eventStorageEngine, errs) + event["storage_engine"] = eventStorageEngine + } + + return event +} + +func mustBeString(key string, data map[string]interface{}, errs map[string]error) string { + emptyIface, exists := data[key] + if !exists { + errs[key] = fmt.Errorf("Key not found") + return "" + } + str, ok := emptyIface.(string) + if !ok { + errs[key] = fmt.Errorf("Expected string, found %T", emptyIface) + return "" + } + return str +} + +func mustBeBool(key string, data map[string]interface{}, errs map[string]error) bool { + emptyIface, exists := data[key] + if !exists { + errs[key] = fmt.Errorf("Key not found") + return false + } + boolean, ok := emptyIface.(bool) + if !ok { + errs[key] = fmt.Errorf("Expected bool, found %T", emptyIface) + return false + } + return boolean +} + +func mustBeInteger(key string, data map[string]interface{}, errs map[string]error) int64 { + emptyIface, exists := data[key] + if !exists { + errs[key] = fmt.Errorf("Key not found") + return 0 + } + switch emptyIface.(type) { + case int64: + return emptyIface.(int64) + case int: + return int64(emptyIface.(int)) + case float64: + return int64(emptyIface.(float64)) + default: + errs[key] = fmt.Errorf("Expected integer, found %T", emptyIface) + return 0 + } +} + +func mustBeTime(key string, data map[string]interface{}, errs map[string]error) common.Time { + emptyIface, exists := data[key] + if !exists { + errs[key] = fmt.Errorf("Key not found") + return common.Time(time.Unix(0, 0)) + } + ts, ok := emptyIface.(time.Time) + if !ok { + errs[key] = fmt.Errorf("Expected date, found %T", emptyIface) + return common.Time(time.Unix(0, 0)) + } + return common.Time(ts) +} + +func removeErroredKeys(event common.MapStr, errs map[string]error) { + for key, err := range errs { + logp.Err("Error on key `%s`: %v", key, err) + delete(event, key) + } +} diff --git a/metricbeat/module/mongodb/status/status.go b/metricbeat/module/mongodb/status/status.go new file mode 100644 index 00000000000..32ab40686c9 --- /dev/null +++ b/metricbeat/module/mongodb/status/status.go @@ -0,0 +1,59 @@ +package status + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/pkg/errors" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" +) + +/* +TODOs: + * add support for username/password + * add metricset for "locks" data + * add a metricset for "metrics" data +*/ + +func init() { + if err := mb.Registry.AddMetricSet("mongodb", "status", New); err != nil { + panic(err) + } +} + +type MetricSet struct { + mb.BaseMetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +func (m *MetricSet) Fetch() (common.MapStr, error) { + + session, err := mgo.DialWithTimeout(m.Host(), m.Module().Config().Timeout) + if err != nil { + return nil, err + } + defer session.Close() + + session.SetMode(mgo.Monotonic, true) + result := map[string]interface{}{} + if err := session.DB("admin").Run(bson.D{{"serverStatus", 1}}, &result); err != nil { + return nil, errors.Wrap(err, "mongodb fetch failed") + } + + return eventMapping(result), nil +} diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go new file mode 100644 index 00000000000..99a39d7fc09 --- /dev/null +++ b/metricbeat/module/mongodb/status/status_integration_test.go @@ -0,0 +1,48 @@ +// +build integration + +package status + +import ( + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + + // Check event fields + current := event["connections"].(common.MapStr)["current"].(int64) + assert.True(t, current >= 0) + + available := event["connections"].(common.MapStr)["available"].(int64) + assert.True(t, available > 0) + + commits := event["journaling"].(common.MapStr)["commits"].(int64) + assert.True(t, commits >= 0) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "mongodb", + "metricsets": []string{"status"}, + "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, + } +} diff --git a/metricbeat/module/mongodb/testing.go b/metricbeat/module/mongodb/testing.go new file mode 100644 index 00000000000..910aaf5b15b --- /dev/null +++ b/metricbeat/module/mongodb/testing.go @@ -0,0 +1,29 @@ +package mongodb + +import "os" + +// Helper functions for testing used in the mongodb metricsets + +// GetEnvHost returns the hostname of the Mongodb server to use for testing. +// It reads the value from the MONGODB_HOST environment variable and returns +// 127.0.0.1 if it is not set. +func GetEnvHost() string { + host := os.Getenv("MONGODB_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +// GetMongodbEnvPort returns the port of the Mongodb server to use for testing. +// It reads the value from the MONGODB_PORT environment variable and returns +// 27017 if it is not set. +func GetEnvPort() string { + port := os.Getenv("MONGODB_PORT") + + if len(port) == 0 { + port = "27017" + } + return port +} diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py new file mode 100644 index 00000000000..b6b03d522e9 --- /dev/null +++ b/metricbeat/tests/system/test_mongodb.py @@ -0,0 +1,38 @@ +import os +import metricbeat +from nose.plugins.attrib import attr + +MONGODB_FIELDS = metricbeat.COMMON_FIELDS + ["mongodb"] + + +class Test(metricbeat.BaseTest): + @attr('integration') + def test_status(self): + """ + MongoDB module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "mongodb", + "metricsets": ["status"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log, "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(MONGODB_FIELDS), evt.keys()) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('MONGODB_HOST', 'localhost') + ':' + + os.getenv('MONGODB_PORT', '27017')]