Skip to content

Commit

Permalink
PMM-12957 pbm metrics collector (#866)
Browse files Browse the repository at this point in the history 
* set up pbm and mongo instances for test

* set up exporter for pbm metrics

* downgrade mongo driver version to fix build

* fix set up scripts for tests

* bump pbm sdk version to fix dependency issue with mongo-tools

* fix linter errors

* fix new linter warnings

* add impl and tests for agent status

* drop direct metric comparison from tests

* use constants for metric value

* fix license header

* re-use createmetric function across methods

* fix linter errors

* use existing replica set for tests

* drop pbm backups directory

* change module version to dev

* use better types for enums

* remove extra check on metrics collection

* fix startup flags

* fix opts for PMM

* do not use direct connection for PBM

* use correct host and port for tests
  • Loading branch information
@idoqo
idoqo authored Aug 26, 2024
1 parent 8fa6455 commit 76f3d4e
Show file tree
Hide file tree
Showing 15 changed files with 478 additions and 55 deletions.
2 changes: 2 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ test-race: env ## Run all tests with race flag.

test-cluster: env ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster
docker compose up --build -d
./docker/scripts/setup-pbm.sh

test-cluster-clean: env ## Stops MongoDB test cluster.
docker compose down --remove-orphans

61 changes: 31 additions & 30 deletions REFERENCE.md
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,34 @@
# Usage Reference

## Flags
|Flag|Description|Example|
|-----|-----|-----|
|-h, \-\-help|Show context-sensitive help||
|--[no-]compatible-mode|Enable old mongodb-exporter compatible metrics||
|--[no-]discovering-mode|Enable autodiscover collections||
|--mongodb.collstats-colls|List of comma separared databases.collections to get $collStats|--mongodb.collstats-colls=db1,db2.col2|
|--mongodb.indexstats-colls|List of comma separared databases.collections to get $indexStats|--mongodb.indexstats-colls=db1.col1,db2.col2|
|--[no-]mongodb.direct-connect|Whether or not a direct connect should be made. Direct connections are not valid if multiple hosts are specified or an SRV URI is used||
|--[no-]mongodb.global-conn-pool|Use global connection pool instead of creating new pool for each http request||
|--mongodb.uri|MongoDB connection URI ($MONGODB_URI)|--mongodb.uri=mongodb://user:pass@127.0.0.1:27017/admin?ssl=true|
|--web.listen-address|Address to listen on for web interface and telemetry|--web.listen-address=":9216"|
|--web.telemetry-path|Metrics expose path|--web.telemetry-path="/metrics"|
|--web.config|Path to the file having Prometheus TLS config for basic auth|--web.config=STRING|
|--web.timeout-offset|Offset to subtract from the timeout in seconds|--web.timeout-offset=1|
|--log.level|Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]|--log.level="error"|
|--collector.diagnosticdata|Enable collecting metrics from getDiagnosticData|
|--collector.replicasetstatus|Enable collecting metrics from replSetGetStatus|
|--collector.dbstats|Enable collecting metrics from dbStats||
|--collector.dbstatsfreestorage|Enable collecting freeStorage metrics from dbStats. If the instance has a large number of collections or indexes, obtaining free space usage data may cause processing delays||
|--collector.topmetrics|Enable collecting metrics from top admin command|
|--collector.currentopmetrics|Enable collecting metrics from currentop admin command|
|--collector.indexstats|Enable collecting metrics from $indexStats|
|--collector.collstats|Enable collecting metrics from $collStats|
|--collect-all|Enable all collectors. Same as specifying all --collector.\<name\>|
|--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \<n\> collections. 0=No limit|
|--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval|
|--collector.profile|Enable collecting metrics from profile|
|--collector.shards|Enable collecting metrics related to Mongo shards|
|--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC ||
|--version|Show version and exit|
| Flag | Description | Example |
|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------|
| -h, \-\-help | Show context-sensitive help | |
| --[no-]compatible-mode | Enable old mongodb-exporter compatible metrics | |
| --[no-]discovering-mode | Enable autodiscover collections | |
| --mongodb.collstats-colls | List of comma separared databases.collections to get $collStats | --mongodb.collstats-colls=db1,db2.col2 |
| --mongodb.indexstats-colls | List of comma separared databases.collections to get $indexStats | --mongodb.indexstats-colls=db1.col1,db2.col2 |
| --[no-]mongodb.direct-connect | Whether or not a direct connect should be made. Direct connections are not valid if multiple hosts are specified or an SRV URI is used | |
| --[no-]mongodb.global-conn-pool | Use global connection pool instead of creating new pool for each http request | |
| --mongodb.uri | MongoDB connection URI ($MONGODB_URI) | --mongodb.uri=mongodb://user:pass@127.0.0.1:27017/admin?ssl=true |
| --web.listen-address | Address to listen on for web interface and telemetry | --web.listen-address=":9216" |
| --web.telemetry-path | Metrics expose path | --web.telemetry-path="/metrics" |
| --web.config | Path to the file having Prometheus TLS config for basic auth | --web.config=STRING |
| --web.timeout-offset | Offset to subtract from the timeout in seconds | --web.timeout-offset=1 |
| --log.level | Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] | --log.level="error" |
| --collector.diagnosticdata | Enable collecting metrics from getDiagnosticData |
| --collector.replicasetstatus | Enable collecting metrics from replSetGetStatus |
| --collector.dbstats | Enable collecting metrics from dbStats | |
| --collector.dbstatsfreestorage | Enable collecting freeStorage metrics from dbStats. If the instance has a large number of collections or indexes, obtaining free space usage data may cause processing delays | |
| --collector.topmetrics | Enable collecting metrics from top admin command |
| --collector.currentopmetrics | Enable collecting metrics from currentop admin command |
| --collector.indexstats | Enable collecting metrics from $indexStats |
| --collector.collstats | Enable collecting metrics from $collStats |
| --collect-all | Enable all collectors. Same as specifying all --collector.\<name\> |
| --collector.collstats-limit=0 | Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \<n\> collections. 0=No limit |
| --collector.profile-time-ts=30 | Set time for scrape slow queries | This interval must be synchronized with the Prometheus scrape interval |
| --collector.profile | Enable collecting metrics from profile |
| --collector.shards | Enable collecting metrics related to Mongo shards |
| --collector.pbm | Enable collecting metrics related to Percona Backup for MongoDB |
| --metrics.overridedescendingindex | Enable descending index name override to replace -1 with _DESC |
| --version | Show version and exit |
57 changes: 48 additions & 9 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '3.8'
services:
mongo-1-1:
container_name: "mongo-1-1"
Expand Down Expand Up @@ -65,8 +64,6 @@ services:
container_name: "mongo-2-2"
build:
dockerfile: ./docker/mongodb-auth.dockerfile
args:
TEST_MONGODB_IMAGE: ${TEST_MONGODB_IMAGE}
environment:
- MONGO_INITDB_ROOT_USERNAME=${TEST_MONGODB_USERNAME:-admin}
- MONGO_INITDB_ROOT_PASSWORD=${TEST_MONGODB_PASSWORD:-admin}
Expand All @@ -80,8 +77,6 @@ services:
container_name: "mongo-2-3"
build:
dockerfile: ./docker/mongodb-auth.dockerfile
args:
TEST_MONGODB_IMAGE: ${TEST_MONGODB_IMAGE}
ports:
- "${TEST_MONGODB_S2_SECONDARY1_PORT:-17005}:27017"
command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile
Expand All @@ -92,8 +87,6 @@ services:
container_name: "mongo-2-1"
build:
dockerfile: ./docker/mongodb-auth.dockerfile
args:
TEST_MONGODB_IMAGE: ${TEST_MONGODB_IMAGE}
ports:
- "${TEST_MONGODB_S2_SECONDARY2_PORT:-17006}:27017"
command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile
Expand All @@ -104,14 +97,57 @@ services:
container_name: "mongo-2-arbiter"
build:
dockerfile: ./docker/mongodb-auth.dockerfile
args:
TEST_MONGODB_IMAGE: ${TEST_MONGODB_IMAGE}
ports:
- "${TEST_MONGODB_S2_ARBITER:-17012}:27017"
command: mongod --replSet rs2 --port 27017 --oplogSize 16 --auth --keyFile=/opt/keyfile
networks:
- rs2

pbm-mongo-2-1:
image: percona/percona-backup-mongodb:2.5.0-multi
container_name: "pbm-mongo-2-1"
restart: on-failure:5
depends_on:
- mongo-2-1
environment:
- PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-1:27017
volumes:
- ./docker/pbm/config:/etc/config
- ./docker/scripts:/scripts
- pbm-backups:/opt/backups
networks:
- rs2

pbm-mongo-2-2:
image: percona/percona-backup-mongodb:2.5.0-multi
container_name: "pbm-mongo-2-2"
restart: on-failure:5
depends_on:
- mongo-2-2
environment:
- PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-2:27017
volumes:
- ./docker/pbm/config:/etc/config
- ./docker/scripts:/scripts
- pbm-backups:/opt/backups
networks:
- rs2

pbm-mongo-2-3:
image: percona/percona-backup-mongodb:2.5.0-multi
container_name: "pbm-mongo-2-3"
restart: on-failure:5
depends_on:
- mongo-2-3
environment:
- PBM_MONGODB_URI=mongodb://admin:admin@mongo-2-3:27017
volumes:
- ./docker/pbm/config:/etc/config
- ./docker/scripts:/scripts
- pbm-backups:/opt/backups
networks:
- rs2

mongo-rs2-setup:
container_name: "mongo-rs2-setup"
image: ${TEST_MONGODB_IMAGE:-mongo:4.2}
Expand Down Expand Up @@ -250,6 +286,9 @@ services:
- ./docker/scripts:/scripts
command: /scripts/run-mongodb-encrypted.sh

volumes:
pbm-backups:

networks:
rs1:
rs2:
Expand Down
3 changes: 2 additions & 1 deletion docker/mongodb-auth.dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
ARG TEST_MONGODB_IMAGE=mongo:4.2
ARG TEST_MONGODB_IMAGE=mongo:4.4
FROM ${TEST_MONGODB_IMAGE}
USER root
COPY docker/secret/keyfile /opt/keyfile
RUN chown mongodb /opt/keyfile && chmod 400 /opt/keyfile && mkdir -p /home/mongodb/ && chown mongodb /home/mongodb
RUN mkdir /opt/backups && touch /opt/backups/.gitkeep && chown mongodb /opt/backups
USER mongodb
4 changes: 4 additions & 0 deletions docker/pbm/config/pbm.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
storage:
type: filesystem
filesystem:
path: /opt/backups/pbm
15 changes: 15 additions & 0 deletions docker/scripts/setup-pbm.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash
docker exec -it --user root pbm-mongo-2-1 bash -c "chown -R mongodb /opt/backups"

# PBM config fails if replica sets are not completely up, so give enough time for both replica sets and pbm agents to be up.
sleep 20

docker exec pbm-mongo-2-1 bash -c "pbm config --file /etc/config/pbm.yaml"

# Wait until agents are restarted after config has been updated
sleep 5

docker exec pbm-mongo-2-1 bash -c "pbm backup"

# Wait for backup to complete
sleep 3
1 change: 1 addition & 0 deletions docker/scripts/setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ arbiter=`getent hosts ${ARBITER} | awk '{ print $1 }'`

username=${MONGO_INITDB_ROOT_USERNAME}
password=${MONGO_INITDB_ROOT_PASSWORD}
backups_dir=${PBM_BACKUPS_DIR}

port=${PORT:-27017}

Expand Down
2 changes: 1 addition & 1 deletion exporter/diagnostic_data_collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ func TestDisconnectedDiagnosticDataCollector(t *testing.T) {
assert.NoError(t, err)

logger := logrus.New()
logger.Out = io.Discard // diable logs in tests
logger.Out = io.Discard // disable logs in tests

ti := labelsGetterMock{}

Expand Down
12 changes: 12 additions & 0 deletions exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ type Opts struct {

EnableOverrideDescendingIndex bool

// Enable metrics for Percona Backup for MongoDB (PBM).
EnablePBMMetrics bool

IndexStatsCollections []string
Logger *logrus.Logger

Expand Down Expand Up @@ -161,6 +164,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
e.opts.EnableCurrentopMetrics = true
e.opts.EnableProfile = true
e.opts.EnableShards = true
e.opts.EnablePBMMetrics = true
}

// arbiter only have isMaster privileges
Expand All @@ -174,6 +178,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
e.opts.EnableCurrentopMetrics = false
e.opts.EnableProfile = false
e.opts.EnableShards = false
e.opts.EnablePBMMetrics = false
}

// If we manually set the collection names we want or auto discovery is set.
Expand Down Expand Up @@ -234,6 +239,11 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
registry.MustRegister(sc)
}

if e.opts.EnablePBMMetrics && requestOpts.EnablePBMMetrics {
pbmc := newPbmCollector(ctx, client, e.opts.URI, e.opts.Logger)
registry.MustRegister(pbmc)
}

return registry
}

Expand Down Expand Up @@ -310,6 +320,8 @@ func (e *Exporter) Handler() http.Handler {
requestOpts.EnableProfile = true
case "shards":
requestOpts.EnableShards = true
case "pbm":
requestOpts.EnablePBMMetrics = true
}
}

Expand Down
Loading

0 comments on commit 76f3d4e

Please sign in to comment.