From 784386159561768b72d630ab1bc01f4efd805bd0 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Wed, 6 Mar 2019 16:39:49 +0100 Subject: [PATCH 1/2] [Metricbeat] Migrate couchbase cluster & pool metricset to new data generation The data.json is now generated based on example JSON documents. In case they change, it means the code change in some way. This will make it much easier to detect changes. Also the basic tests for the cluster are now covered by the new testing. The error handling moved over to the new error interface for simplicity. --- metricbeat/mb/testing/data_test.go | 2 + .../module/couchbase/cluster/_meta/data.json | 24 ++-- .../cluster/_meta/testdata/4.5.1.json | 1 + .../_meta/testdata/4.5.1.json-expected.json | 81 +++++++++++ .../cluster/_meta/testdata/config.yml | 2 + .../testdata/docs.json} | 0 .../_meta/testdata/docs.json-expected.json | 81 +++++++++++ .../module/couchbase/cluster/cluster.go | 10 +- .../cluster/cluster_integration_test.go | 15 -- .../module/couchbase/cluster/cluster_test.go | 120 ---------------- .../module/couchbase/node/_meta/data.json | 36 +++-- .../couchbase/node/_meta/testdata/4.5.1.json | 1 + .../_meta/testdata/4.5.1.json-expected.json | 89 ++++++++++++ .../couchbase/node/_meta/testdata/config.yml | 2 + .../testdata/docs.json} | 0 .../_meta/testdata/docs.json-expected.json | 89 ++++++++++++ metricbeat/module/couchbase/node/node.go | 11 +- .../couchbase/node/node_integration_test.go | 15 -- metricbeat/module/couchbase/node/node_test.go | 133 ------------------ 19 files changed, 380 insertions(+), 332 deletions(-) create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/config.yml rename metricbeat/module/couchbase/cluster/{testdata/sample_response.json => _meta/testdata/docs.json} (100%) create mode 100644 metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json delete mode 100644 metricbeat/module/couchbase/cluster/cluster_test.go create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/config.yml rename metricbeat/module/couchbase/node/{testdata/sample_response.json => _meta/testdata/docs.json} (100%) create mode 100644 metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json delete mode 100644 metricbeat/module/couchbase/node/node_test.go diff --git a/metricbeat/mb/testing/data_test.go b/metricbeat/mb/testing/data_test.go index 803bfec6d3c3..199dc28b3936 100644 --- a/metricbeat/mb/testing/data_test.go +++ b/metricbeat/mb/testing/data_test.go @@ -36,6 +36,8 @@ import ( "github.com/elastic/beats/metricbeat/mb" // TODO: generate include file for these tests automatically moving forward + _ "github.com/elastic/beats/metricbeat/module/couchbase/cluster" + _ "github.com/elastic/beats/metricbeat/module/couchbase/node" _ "github.com/elastic/beats/metricbeat/module/kibana/status" _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" _ "github.com/elastic/beats/metricbeat/module/php_fpm/process" diff --git a/metricbeat/module/couchbase/cluster/_meta/data.json b/metricbeat/module/couchbase/cluster/_meta/data.json index 461df930f17d..9ee9137dd7bd 100644 --- a/metricbeat/module/couchbase/cluster/_meta/data.json +++ b/metricbeat/module/couchbase/cluster/_meta/data.json @@ -1,29 +1,25 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "couchbase": { "cluster": { "hdd": { "free": { - "bytes": 48341763359 + "bytes": 46902679716 }, "quota": { "total": { - "bytes": 185929859072 + "bytes": 63381999616 } }, "total": { - "bytes": 185929859072 + "bytes": 63381999616 }, "used": { "by_data": { - "bytes": 23665027 + "bytes": 16369010 }, "value": { - "bytes": 137588095713 + "bytes": 16479319900 } } }, @@ -56,14 +52,14 @@ } }, "total": { - "bytes": 16683503616 + "bytes": 8359174144 }, "used": { "by_data": { - "bytes": 53961776 + "bytes": 53962016 }, "value": { - "bytes": 14443474944 + "bytes": 8004751360 } } } @@ -78,7 +74,7 @@ "name": "cluster" }, "service": { - "address": "localhost:8091", + "address": "127.0.0.1:55555", "type": "couchbase" } } \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json new file mode 100644 index 000000000000..d620439f1382 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json @@ -0,0 +1 @@ +{"storageTotals":{"ram":{"total":6246748160,"quotaTotal":314572800,"quotaUsed":104857600,"used":3155247104,"usedByData":53962032,"quotaUsedPerNode":104857600,"quotaTotalPerNode":314572800},"hdd":{"total":62725623808,"quotaTotal":62725623808,"used":5645306142,"usedByData":17923953,"free":57080317666}},"ftsMemoryQuota":512,"indexMemoryQuota":300,"memoryQuota":300,"name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9179f751ee7308159a00974c5da5229f","nodes":[{"systemStats":{"cpu_utilization_rate":5.670103092783505,"swap_total":1073737728,"swap_used":0,"mem_total":6246748160,"mem_free":5239775232},"interestingStats":{"cmd_get":0,"couch_docs_actual_disk_size":17154348,"couch_docs_data_size":17140736,"couch_spatial_data_size":0,"couch_spatial_disk_size":0,"couch_views_actual_disk_size":769605,"couch_views_data_size":769605,"curr_items":7303,"curr_items_tot":7303,"ep_bg_fetched":0,"get_hits":0,"mem_used":53962032,"ops":0,"vb_replica_curr_items":0},"uptime":"30","memoryTotal":6246748160,"memoryFree":5239775232,"mcdMemoryReserved":4765,"mcdMemoryAllocated":4765,"couchApiBase":"http://172.17.0.2:8092/","couchApiBaseHTTPS":"https://172.17.0.2:18092/","otpCookie":"snyykghcidiksmqs","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@127.0.0.1","thisNode":true,"hostname":"172.17.0.2:8091","clusterCompatibility":262149,"version":"4.5.1-2844-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"proxy":11211,"direct":11210},"services":["index","kv","n1ql"]}],"buckets":{"uri":"/pools/default/buckets?v=42234267&uuid=9179f751ee7308159a00974c5da5229f","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"},"remoteClusters":{"uri":"/pools/default/remoteClusters?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/pools/default/remoteClusters?just_validate=1"},"controllers":{"addNode":{"uri":"/controller/addNodeV2?uuid=9179f751ee7308159a00974c5da5229f"},"rebalance":{"uri":"/controller/rebalance?uuid=9179f751ee7308159a00974c5da5229f"},"failOver":{"uri":"/controller/failOver?uuid=9179f751ee7308159a00974c5da5229f"},"startGracefulFailover":{"uri":"/controller/startGracefulFailover?uuid=9179f751ee7308159a00974c5da5229f"},"reAddNode":{"uri":"/controller/reAddNode?uuid=9179f751ee7308159a00974c5da5229f"},"reFailOver":{"uri":"/controller/reFailOver?uuid=9179f751ee7308159a00974c5da5229f"},"ejectNode":{"uri":"/controller/ejectNode?uuid=9179f751ee7308159a00974c5da5229f"},"setRecoveryType":{"uri":"/controller/setRecoveryType?uuid=9179f751ee7308159a00974c5da5229f"},"setAutoCompaction":{"uri":"/controller/setAutoCompaction?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/setAutoCompaction?just_validate=1"},"clusterLogsCollection":{"startURI":"/controller/startLogsCollection?uuid=9179f751ee7308159a00974c5da5229f","cancelURI":"/controller/cancelLogsCollection?uuid=9179f751ee7308159a00974c5da5229f"},"replication":{"createURI":"/controller/createReplication?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/createReplication?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9179f751ee7308159a00974c5da5229f","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":{"percentage":30,"size":"undefined"},"viewFragmentationThreshold":{"percentage":30,"size":"undefined"},"indexCompactionMode":"circular","indexCircularCompaction":{"daysOfWeek":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","interval":{"fromHour":0,"toHour":0,"fromMinute":0,"toMinute":0,"abortOutside":false}},"indexFragmentationThreshold":{"percentage":30}},"tasks":{"uri":"/pools/default/tasks?v=74113599"},"counters":{},"indexStatusURI":"/indexStatus?v=21137658","checkPermissionsURI":"/pools/default/checkPermissions?v=11181330","serverGroupsUri":"/pools/default/serverGroups?v=52184775"} diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json new file mode 100644 index 000000000000..8dbbb27b4198 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/4.5.1.json-expected.json @@ -0,0 +1,81 @@ +[ + { + "couchbase": { + "cluster": { + "hdd": { + "free": { + "bytes": 57080317666 + }, + "quota": { + "total": { + "bytes": 62725623808 + } + }, + "total": { + "bytes": 62725623808 + }, + "used": { + "by_data": { + "bytes": 17923953 + }, + "value": { + "bytes": 5645306142 + } + } + }, + "max_bucket_count": 10, + "quota": { + "index_memory": { + "mb": 300 + }, + "memory": { + "mb": 300 + } + }, + "ram": { + "quota": { + "total": { + "per_node": { + "bytes": 314572800 + }, + "value": { + "bytes": 314572800 + } + }, + "used": { + "per_node": { + "bytes": 104857600 + }, + "value": { + "bytes": 104857600 + } + } + }, + "total": { + "bytes": 6246748160 + }, + "used": { + "by_data": { + "bytes": 53962032 + }, + "value": { + "bytes": 3155247104 + } + } + } + } + }, + "event": { + "dataset": "couchbase.cluster", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "cluster" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml b/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml new file mode 100644 index 000000000000..1351708ffb0c --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/pools/default" diff --git a/metricbeat/module/couchbase/cluster/testdata/sample_response.json b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json similarity index 100% rename from metricbeat/module/couchbase/cluster/testdata/sample_response.json rename to metricbeat/module/couchbase/cluster/_meta/testdata/docs.json diff --git a/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json new file mode 100644 index 000000000000..093aaed8a871 --- /dev/null +++ b/metricbeat/module/couchbase/cluster/_meta/testdata/docs.json-expected.json @@ -0,0 +1,81 @@ +[ + { + "couchbase": { + "cluster": { + "hdd": { + "free": { + "bytes": 46902679716 + }, + "quota": { + "total": { + "bytes": 63381999616 + } + }, + "total": { + "bytes": 63381999616 + }, + "used": { + "by_data": { + "bytes": 16369010 + }, + "value": { + "bytes": 16479319900 + } + } + }, + "max_bucket_count": 10, + "quota": { + "index_memory": { + "mb": 300 + }, + "memory": { + "mb": 300 + } + }, + "ram": { + "quota": { + "total": { + "per_node": { + "bytes": 314572800 + }, + "value": { + "bytes": 314572800 + } + }, + "used": { + "per_node": { + "bytes": 104857600 + }, + "value": { + "bytes": 104857600 + } + } + }, + "total": { + "bytes": 8359174144 + }, + "used": { + "by_data": { + "bytes": 53962016 + }, + "value": { + "bytes": 8004751360 + } + } + } + } + }, + "event": { + "dataset": "couchbase.cluster", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "cluster" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/cluster/cluster.go b/metricbeat/module/couchbase/cluster/cluster.go index 735ef1599307..e2366b3f2722 100644 --- a/metricbeat/module/couchbase/cluster/cluster.go +++ b/metricbeat/module/couchbase/cluster/cluster.go @@ -18,7 +18,6 @@ package cluster import ( - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -36,8 +35,6 @@ var ( }.Build() ) -var logger = logp.NewLogger("couchbase.cluster") - // init registers the MetricSet with the central registry. // The New method will be called after the setup of the module and before starting to fetch data func init() { @@ -68,15 +65,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { content, err := m.http.FetchContent() if err != nil { - logger.Error(err) - reporter.Error(err) - return + return err } reporter.Event(mb.Event{ MetricSetFields: eventMapping(content), }) + return nil } diff --git a/metricbeat/module/couchbase/cluster/cluster_integration_test.go b/metricbeat/module/couchbase/cluster/cluster_integration_test.go index 72a94a530d31..dd1812388a58 100644 --- a/metricbeat/module/couchbase/cluster/cluster_integration_test.go +++ b/metricbeat/module/couchbase/cluster/cluster_integration_test.go @@ -42,21 +42,6 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "couchbase", diff --git a/metricbeat/module/couchbase/cluster/cluster_test.go b/metricbeat/module/couchbase/cluster/cluster_test.go deleted file mode 100644 index d828f3509a77..000000000000 --- a/metricbeat/module/couchbase/cluster/cluster_test.go +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package cluster - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" -) - -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("./testdata/") - assert.NoError(t, err) - - // response is a raw response from a couchbase - response, err := ioutil.ReadFile(absPath + "/sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchbase", - "metricsets": []string{"cluster"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2(t, config) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - hdd := event["hdd"].(common.MapStr) - hdd_free := hdd["free"].(common.MapStr) - assert.EqualValues(t, 46902679716, hdd_free["bytes"]) - - hdd_total := hdd["total"].(common.MapStr) - assert.EqualValues(t, 63381999616, hdd_total["bytes"]) - - hdd_used := hdd["used"].(common.MapStr) - hdd_used_value := hdd_used["value"].(common.MapStr) - assert.EqualValues(t, 16479319900, hdd_used_value["bytes"]) - - hdd_used_by_data := hdd_used["by_data"].(common.MapStr) - assert.EqualValues(t, 16369010, hdd_used_by_data["bytes"]) - - hdd_quota := hdd["quota"].(common.MapStr) - hdd_quota_total := hdd_quota["total"].(common.MapStr) - assert.EqualValues(t, 63381999616, hdd_quota_total["bytes"]) - - assert.EqualValues(t, 10, event["max_bucket_count"]) - - quota := event["quota"].(common.MapStr) - quota_index_memory := quota["index_memory"].(common.MapStr) - assert.EqualValues(t, 300, quota_index_memory["mb"]) - - quota_memory := quota["memory"].(common.MapStr) - assert.EqualValues(t, 300, quota_memory["mb"]) - - ram := event["ram"].(common.MapStr) - - ram_quota := ram["quota"].(common.MapStr) - - ram_quota_total := ram_quota["total"].(common.MapStr) - ram_quota_total_value := ram_quota_total["value"].(common.MapStr) - assert.EqualValues(t, 314572800, ram_quota_total_value["bytes"]) - - ram_quota_total_per_node := ram_quota_total["per_node"].(common.MapStr) - assert.EqualValues(t, 314572800, ram_quota_total_per_node["bytes"]) - - ram_quota_used := ram_quota["used"].(common.MapStr) - ram_quota_used_value := ram_quota_used["value"].(common.MapStr) - assert.EqualValues(t, 104857600, ram_quota_used_value["bytes"]) - - ram_quota_used_per_node := ram_quota_used["per_node"].(common.MapStr) - assert.EqualValues(t, 104857600, ram_quota_used_per_node["bytes"]) - - ram_total := ram["total"].(common.MapStr) - assert.EqualValues(t, 8359174144, ram_total["bytes"]) - - ram_used := ram["used"].(common.MapStr) - ram_used_value := ram_used["value"].(common.MapStr) - assert.EqualValues(t, 8004751360, ram_used_value["bytes"]) - - ram_used_by_data := ram_used["by_data"].(common.MapStr) - assert.EqualValues(t, 53962016, ram_used_by_data["bytes"]) -} diff --git a/metricbeat/module/couchbase/node/_meta/data.json b/metricbeat/module/couchbase/node/_meta/data.json index 750af9accd9a..58cc3b95835a 100644 --- a/metricbeat/module/couchbase/node/_meta/data.json +++ b/metricbeat/module/couchbase/node/_meta/data.json @@ -1,19 +1,15 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "couchbase": { "node": { "cmd_get": 0, "couch": { "docs": { "data_size": { - "bytes": 19151872 + "bytes": 9792512 }, "disk_size": { - "bytes": 22895585 + "bytes": 13563791 } }, "spatial": { @@ -26,15 +22,15 @@ }, "views": { "data_size": { - "bytes": 769446 + "bytes": 2805219 }, "disk_size": { - "bytes": 769446 + "bytes": 2805219 } } }, "cpu_utilization_rate": { - "pct": 19.64735516372796 + "pct": 29.64705882352941 }, "current_items": { "total": 7303, @@ -42,37 +38,37 @@ }, "ep_bg_fetched": 0, "get_hits": 0, - "hostname": "172.21.0.2:8091", + "hostname": "172.17.0.2:8091", "mcd_memory": { "allocated": { - "bytes": 12728 + "bytes": 6377 }, "reserved": { - "bytes": 12728 + "bytes": 6377 } }, "memory": { "free": { - "bytes": 7447130112 + "bytes": 4678324224 }, "total": { - "bytes": 16683503616 + "bytes": 8359174144 }, "used": { - "bytes": 53961776 + "bytes": 53962016 } }, "ops": 0, "swap": { "total": { - "bytes": 2147479552 + "bytes": 4189057024 }, "used": { - "bytes": 0 + "bytes": 135168 } }, "uptime": { - "sec": 10612 + "sec": 7260 }, "vb_replica_curr_items": 0 } @@ -86,7 +82,7 @@ "name": "node" }, "service": { - "address": "localhost:8091", + "address": "127.0.0.1:55555", "type": "couchbase" } } \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json new file mode 100644 index 000000000000..d620439f1382 --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json @@ -0,0 +1 @@ +{"storageTotals":{"ram":{"total":6246748160,"quotaTotal":314572800,"quotaUsed":104857600,"used":3155247104,"usedByData":53962032,"quotaUsedPerNode":104857600,"quotaTotalPerNode":314572800},"hdd":{"total":62725623808,"quotaTotal":62725623808,"used":5645306142,"usedByData":17923953,"free":57080317666}},"ftsMemoryQuota":512,"indexMemoryQuota":300,"memoryQuota":300,"name":"default","alerts":[],"alertsSilenceURL":"/controller/resetAlerts?token=0&uuid=9179f751ee7308159a00974c5da5229f","nodes":[{"systemStats":{"cpu_utilization_rate":5.670103092783505,"swap_total":1073737728,"swap_used":0,"mem_total":6246748160,"mem_free":5239775232},"interestingStats":{"cmd_get":0,"couch_docs_actual_disk_size":17154348,"couch_docs_data_size":17140736,"couch_spatial_data_size":0,"couch_spatial_disk_size":0,"couch_views_actual_disk_size":769605,"couch_views_data_size":769605,"curr_items":7303,"curr_items_tot":7303,"ep_bg_fetched":0,"get_hits":0,"mem_used":53962032,"ops":0,"vb_replica_curr_items":0},"uptime":"30","memoryTotal":6246748160,"memoryFree":5239775232,"mcdMemoryReserved":4765,"mcdMemoryAllocated":4765,"couchApiBase":"http://172.17.0.2:8092/","couchApiBaseHTTPS":"https://172.17.0.2:18092/","otpCookie":"snyykghcidiksmqs","clusterMembership":"active","recoveryType":"none","status":"healthy","otpNode":"ns_1@127.0.0.1","thisNode":true,"hostname":"172.17.0.2:8091","clusterCompatibility":262149,"version":"4.5.1-2844-enterprise","os":"x86_64-unknown-linux-gnu","ports":{"sslProxy":11214,"httpsMgmt":18091,"httpsCAPI":18092,"proxy":11211,"direct":11210},"services":["index","kv","n1ql"]}],"buckets":{"uri":"/pools/default/buckets?v=42234267&uuid=9179f751ee7308159a00974c5da5229f","terseBucketsBase":"/pools/default/b/","terseStreamingBucketsBase":"/pools/default/bs/"},"remoteClusters":{"uri":"/pools/default/remoteClusters?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/pools/default/remoteClusters?just_validate=1"},"controllers":{"addNode":{"uri":"/controller/addNodeV2?uuid=9179f751ee7308159a00974c5da5229f"},"rebalance":{"uri":"/controller/rebalance?uuid=9179f751ee7308159a00974c5da5229f"},"failOver":{"uri":"/controller/failOver?uuid=9179f751ee7308159a00974c5da5229f"},"startGracefulFailover":{"uri":"/controller/startGracefulFailover?uuid=9179f751ee7308159a00974c5da5229f"},"reAddNode":{"uri":"/controller/reAddNode?uuid=9179f751ee7308159a00974c5da5229f"},"reFailOver":{"uri":"/controller/reFailOver?uuid=9179f751ee7308159a00974c5da5229f"},"ejectNode":{"uri":"/controller/ejectNode?uuid=9179f751ee7308159a00974c5da5229f"},"setRecoveryType":{"uri":"/controller/setRecoveryType?uuid=9179f751ee7308159a00974c5da5229f"},"setAutoCompaction":{"uri":"/controller/setAutoCompaction?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/setAutoCompaction?just_validate=1"},"clusterLogsCollection":{"startURI":"/controller/startLogsCollection?uuid=9179f751ee7308159a00974c5da5229f","cancelURI":"/controller/cancelLogsCollection?uuid=9179f751ee7308159a00974c5da5229f"},"replication":{"createURI":"/controller/createReplication?uuid=9179f751ee7308159a00974c5da5229f","validateURI":"/controller/createReplication?just_validate=1"}},"rebalanceStatus":"none","rebalanceProgressUri":"/pools/default/rebalanceProgress","stopRebalanceUri":"/controller/stopRebalance?uuid=9179f751ee7308159a00974c5da5229f","nodeStatusesUri":"/nodeStatuses","maxBucketCount":10,"autoCompactionSettings":{"parallelDBAndViewCompaction":false,"databaseFragmentationThreshold":{"percentage":30,"size":"undefined"},"viewFragmentationThreshold":{"percentage":30,"size":"undefined"},"indexCompactionMode":"circular","indexCircularCompaction":{"daysOfWeek":"Sunday,Monday,Tuesday,Wednesday,Thursday,Friday,Saturday","interval":{"fromHour":0,"toHour":0,"fromMinute":0,"toMinute":0,"abortOutside":false}},"indexFragmentationThreshold":{"percentage":30}},"tasks":{"uri":"/pools/default/tasks?v=74113599"},"counters":{},"indexStatusURI":"/indexStatus?v=21137658","checkPermissionsURI":"/pools/default/checkPermissions?v=11181330","serverGroupsUri":"/pools/default/serverGroups?v=52184775"} diff --git a/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json new file mode 100644 index 000000000000..5d4ea32844aa --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/4.5.1.json-expected.json @@ -0,0 +1,89 @@ +[ + { + "couchbase": { + "node": { + "cmd_get": 0, + "couch": { + "docs": { + "data_size": { + "bytes": 17140736 + }, + "disk_size": { + "bytes": 17154348 + } + }, + "spatial": { + "data_size": { + "bytes": 0 + }, + "disk_size": { + "bytes": 0 + } + }, + "views": { + "data_size": { + "bytes": 769605 + }, + "disk_size": { + "bytes": 769605 + } + } + }, + "cpu_utilization_rate": { + "pct": 5.670103092783505 + }, + "current_items": { + "total": 7303, + "value": 7303 + }, + "ep_bg_fetched": 0, + "get_hits": 0, + "hostname": "172.17.0.2:8091", + "mcd_memory": { + "allocated": { + "bytes": 4765 + }, + "reserved": { + "bytes": 4765 + } + }, + "memory": { + "free": { + "bytes": 5239775232 + }, + "total": { + "bytes": 6246748160 + }, + "used": { + "bytes": 53962032 + } + }, + "ops": 0, + "swap": { + "total": { + "bytes": 1073737728 + }, + "used": { + "bytes": 0 + } + }, + "uptime": { + "sec": 30 + }, + "vb_replica_curr_items": 0 + } + }, + "event": { + "dataset": "couchbase.node", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "node" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/_meta/testdata/config.yml b/metricbeat/module/couchbase/node/_meta/testdata/config.yml new file mode 100644 index 000000000000..1351708ffb0c --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/config.yml @@ -0,0 +1,2 @@ +type: http +url: "/pools/default" diff --git a/metricbeat/module/couchbase/node/testdata/sample_response.json b/metricbeat/module/couchbase/node/_meta/testdata/docs.json similarity index 100% rename from metricbeat/module/couchbase/node/testdata/sample_response.json rename to metricbeat/module/couchbase/node/_meta/testdata/docs.json diff --git a/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json b/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json new file mode 100644 index 000000000000..c43091bbf5b0 --- /dev/null +++ b/metricbeat/module/couchbase/node/_meta/testdata/docs.json-expected.json @@ -0,0 +1,89 @@ +[ + { + "couchbase": { + "node": { + "cmd_get": 0, + "couch": { + "docs": { + "data_size": { + "bytes": 9792512 + }, + "disk_size": { + "bytes": 13563791 + } + }, + "spatial": { + "data_size": { + "bytes": 0 + }, + "disk_size": { + "bytes": 0 + } + }, + "views": { + "data_size": { + "bytes": 2805219 + }, + "disk_size": { + "bytes": 2805219 + } + } + }, + "cpu_utilization_rate": { + "pct": 29.64705882352941 + }, + "current_items": { + "total": 7303, + "value": 7303 + }, + "ep_bg_fetched": 0, + "get_hits": 0, + "hostname": "172.17.0.2:8091", + "mcd_memory": { + "allocated": { + "bytes": 6377 + }, + "reserved": { + "bytes": 6377 + } + }, + "memory": { + "free": { + "bytes": 4678324224 + }, + "total": { + "bytes": 8359174144 + }, + "used": { + "bytes": 53962016 + } + }, + "ops": 0, + "swap": { + "total": { + "bytes": 4189057024 + }, + "used": { + "bytes": 135168 + } + }, + "uptime": { + "sec": 7260 + }, + "vb_replica_curr_items": 0 + } + }, + "event": { + "dataset": "couchbase.node", + "duration": 115000, + "module": "couchbase" + }, + "metricset": { + "name": "node" + }, + "service": { + "address": "127.0.0.1:55555", + "type": "couchbase" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/couchbase/node/node.go b/metricbeat/module/couchbase/node/node.go index 84a0dad9fd9e..c06da22b747e 100644 --- a/metricbeat/module/couchbase/node/node.go +++ b/metricbeat/module/couchbase/node/node.go @@ -18,7 +18,6 @@ package node import ( - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -36,8 +35,6 @@ var ( }.Build() ) -var logger = logp.NewLogger("couchbase.node") - // init registers the MetricSet with the central registry. // The New method will be called after the setup of the module and before starting to fetch data func init() { @@ -68,12 +65,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { content, err := m.http.FetchContent() if err != nil { - logger.Error(err) - reporter.Error(err) - return + return err } events := eventsMapping(content) @@ -81,5 +76,5 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { reporter.Event(mb.Event{MetricSetFields: event}) } - return + return nil } diff --git a/metricbeat/module/couchbase/node/node_integration_test.go b/metricbeat/module/couchbase/node/node_integration_test.go index 9513e91cf7a0..3612e63238a2 100644 --- a/metricbeat/module/couchbase/node/node_integration_test.go +++ b/metricbeat/module/couchbase/node/node_integration_test.go @@ -42,21 +42,6 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { - t.Fatal("write", err) - } -} - func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "couchbase", diff --git a/metricbeat/module/couchbase/node/node_test.go b/metricbeat/module/couchbase/node/node_test.go deleted file mode 100644 index 293195de01ff..000000000000 --- a/metricbeat/module/couchbase/node/node_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package node - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - - "github.com/elastic/beats/libbeat/common" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("./testdata/") - assert.NoError(t, err) - - // response is a raw response from a couchbase - response, err := ioutil.ReadFile(absPath + "/sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "couchbase", - "metricsets": []string{"node"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2(t, config) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - assert.EqualValues(t, 0, event["cmd_get"]) - - couch := event["couch"].(common.MapStr) - - couch_docs := couch["docs"].(common.MapStr) - couch_docs_disk_size := couch_docs["disk_size"].(common.MapStr) - assert.EqualValues(t, 13563791, couch_docs_disk_size["bytes"]) - - couch_docs_data_size := couch_docs["data_size"].(common.MapStr) - assert.EqualValues(t, 9792512, couch_docs_data_size["bytes"]) - - couch_spatial := couch["spatial"].(common.MapStr) - couch_spatial_data_size := couch_spatial["data_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spatial_data_size["bytes"]) - - couch_spatial_disk_size := couch_spatial["disk_size"].(common.MapStr) - assert.EqualValues(t, 0, couch_spatial_disk_size["bytes"]) - - couch_views := couch["views"].(common.MapStr) - couch_views_disk_size := couch_views["disk_size"].(common.MapStr) - assert.EqualValues(t, 2805219, couch_views_disk_size["bytes"]) - - couch_views_data_size := couch_views["data_size"].(common.MapStr) - assert.EqualValues(t, 2805219, couch_views_data_size["bytes"]) - - event_cpu_utilization_rate := event["cpu_utilization_rate"].(common.MapStr) - assert.EqualValues(t, 29.64705882352941, event_cpu_utilization_rate["pct"]) - - current_items := event["current_items"].(common.MapStr) - assert.EqualValues(t, 7303, current_items["value"]) - assert.EqualValues(t, 7303, current_items["total"]) - - assert.EqualValues(t, 0, event["ep_bg_fetched"]) - assert.EqualValues(t, 0, event["get_hits"]) - assert.Equal(t, "172.17.0.2:8091", event["hostname"]) - - mcd_memory := event["mcd_memory"].(common.MapStr) - mcd_memory_reserved := mcd_memory["reserved"].(common.MapStr) - assert.EqualValues(t, 6377, mcd_memory_reserved["bytes"]) - - mcd_memory_allocated := mcd_memory["allocated"].(common.MapStr) - assert.EqualValues(t, 6377, mcd_memory_allocated["bytes"]) - - memory := event["memory"].(common.MapStr) - memory_total := memory["total"].(common.MapStr) - assert.EqualValues(t, 8359174144, memory_total["bytes"]) - - memory_free := memory["free"].(common.MapStr) - assert.EqualValues(t, 4678324224, memory_free["bytes"]) - - memory_used := memory["used"].(common.MapStr) - assert.EqualValues(t, 53962016, memory_used["bytes"]) - - assert.EqualValues(t, 0, event["ops"]) - - swap := event["swap"].(common.MapStr) - swap_total := swap["total"].(common.MapStr) - assert.EqualValues(t, 4189057024, swap_total["bytes"]) - - swap_used := swap["used"].(common.MapStr) - assert.EqualValues(t, 135168, swap_used["bytes"]) - - event_uptime := event["uptime"].(common.MapStr) - assert.EqualValues(t, 7260, event_uptime["sec"]) - - assert.EqualValues(t, 0, event["vb_replica_curr_items"]) -} From 0a551a75f4392d2726f08e1b74ddcdaf91deb6cd Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Thu, 7 Mar 2019 09:07:17 +0100 Subject: [PATCH 2/2] remove logger changes --- metricbeat/module/couchbase/cluster/cluster.go | 10 +++++++--- metricbeat/module/couchbase/node/node.go | 11 ++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/couchbase/cluster/cluster.go b/metricbeat/module/couchbase/cluster/cluster.go index e2366b3f2722..735ef1599307 100644 --- a/metricbeat/module/couchbase/cluster/cluster.go +++ b/metricbeat/module/couchbase/cluster/cluster.go @@ -18,6 +18,7 @@ package cluster import ( + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -35,6 +36,8 @@ var ( }.Build() ) +var logger = logp.NewLogger("couchbase.cluster") + // init registers the MetricSet with the central registry. // The New method will be called after the setup of the module and before starting to fetch data func init() { @@ -65,14 +68,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { content, err := m.http.FetchContent() if err != nil { - return err + logger.Error(err) + reporter.Error(err) + return } reporter.Event(mb.Event{ MetricSetFields: eventMapping(content), }) - return nil } diff --git a/metricbeat/module/couchbase/node/node.go b/metricbeat/module/couchbase/node/node.go index c06da22b747e..84a0dad9fd9e 100644 --- a/metricbeat/module/couchbase/node/node.go +++ b/metricbeat/module/couchbase/node/node.go @@ -18,6 +18,7 @@ package node import ( + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -35,6 +36,8 @@ var ( }.Build() ) +var logger = logp.NewLogger("couchbase.node") + // init registers the MetricSet with the central registry. // The New method will be called after the setup of the module and before starting to fetch data func init() { @@ -65,10 +68,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { content, err := m.http.FetchContent() if err != nil { - return err + logger.Error(err) + reporter.Error(err) + return } events := eventsMapping(content) @@ -76,5 +81,5 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { reporter.Event(mb.Event{MetricSetFields: event}) } - return nil + return }