PMM-12548 Mongolog as new MongoDB query source.

admin/cmd/bootstrap.go

@@ -143,6 +143,7 @@ func getDefaultKongOptions(appName string) []kong.Option {
 
 	mongoDBQuerySources := []string{
 		management.MongodbQuerySourceProfiler,
+		management.MongodbQuerySourceMongolog,
 		management.MongodbQuerySourceNone,
 	}
 
@@ -159,13 +160,13 @@ func getDefaultKongOptions(appName string) []kong.Option {
 			"nodeIp":                       nodeinfo.PublicAddress,
 			"nodeTypeDefault":              nodeTypeDefault,
 			"hostname":                     hostname,
-			"serviceTypesEnum":             strings.Join(management.AllServiceTypesKeys, ","),
+			"serviceTypesEnum":             strings.Join(management.AllServiceTypesKeys, ", "),
 			"defaultMachineID":             defaultMachineID,
 			"distro":                       nodeinfo.Distro,
-			"metricsModesEnum":             strings.Join(management.MetricsModes, ","),
-			"mysqlQuerySourcesEnum":        strings.Join(mysqlQuerySources, ","),
+			"metricsModesEnum":             strings.Join(management.MetricsModes, ", "),
+			"mysqlQuerySourcesEnum":        strings.Join(mysqlQuerySources, ", "),
 			"mysqlQuerySourceDefault":      mysqlQuerySources[0],
-			"mongoDbQuerySourcesEnum":      strings.Join(mongoDBQuerySources, ","),
+			"mongoDbQuerySourcesEnum":      strings.Join(mongoDBQuerySources, ", "),
 			"mongoDbQuerySourceDefault":    mongoDBQuerySources[0],
 			"externalDefaultServiceName":   management.DefaultServiceNameSuffix,
 			"externalDefaultGroupExporter": management.DefaultGroupExternalExporter,

admin/commands/inventory/add_agent_qan_mongodb_mongolog_agent.go

@@ -0,0 +1,110 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package inventory
+
+import (
+	"github.com/percona/pmm/admin/commands"
+	"github.com/percona/pmm/admin/pkg/flags"
+	"github.com/percona/pmm/api/inventory/v1/json/client"
+	agents "github.com/percona/pmm/api/inventory/v1/json/client/agents_service"
+)
+
+var addAgentQANMongoDBMongologAgentResultT = commands.ParseTemplate(`
+QAN MongoDB Mongolog agent added.
+Agent ID              : {{ .Agent.AgentID }}
+PMM-Agent ID          : {{ .Agent.PMMAgentID }}
+Service ID            : {{ .Agent.ServiceID }}
+Username              : {{ .Agent.Username }}
+TLS enabled           : {{ .Agent.TLS }}
+Skip TLS verification : {{ .Agent.TLSSkipVerify }}
+
+Status                : {{ .Agent.Status }}
+Disabled              : {{ .Agent.Disabled }}
+Custom labels         : {{ .Agent.CustomLabels }}
+`)
+
+type addAgentQANMongoDBMongologAgentResult struct {
+	Agent *agents.AddAgentOKBodyQANMongodbMongologAgent `json:"qan_mongodb_mongolog_agent"`
+}
+
+func (res *addAgentQANMongoDBMongologAgentResult) Result() {}
+
+func (res *addAgentQANMongoDBMongologAgentResult) String() string {
+	return commands.RenderTemplate(addAgentQANMongoDBMongologAgentResultT, res)
+}
+
+// AddAgentQANMongoDBMongologAgentCommand is used by Kong for CLI flags and commands.
+type AddAgentQANMongoDBMongologAgentCommand struct {
+	PMMAgentID                    string            `arg:"" help:"The pmm-agent identifier which runs this instance"`
+	ServiceID                     string            `arg:"" help:"Service identifier"`
+	Username                      string            `arg:"" optional:"" help:"MongoDB username for scraping metrics"`
+	Password                      string            `help:"MongoDB password for scraping metrics"`
+	CustomLabels                  map[string]string `mapsep:"," help:"Custom user-assigned labels"`
+	SkipConnectionCheck           bool              `help:"Skip connection check"`
+	MaxQueryLength                int32             `placeholder:"NUMBER" help:"Limit query length in QAN (default: server-defined; -1: no limit)"`
+	DisableQueryExamples          bool              `name:"disable-queryexamples" help:"Disable collection of query examples"`
+	TLS                           bool              `help:"Use TLS to connect to the database"`
+	TLSSkipVerify                 bool              `help:"Skip TLS certificates validation"`
+	TLSCertificateKeyFile         string            `help:"Path to TLS certificate PEM file"`
+	TLSCertificateKeyFilePassword string            `help:"Password for certificate"`
+	TLSCaFile                     string            `help:"Path to certificate authority file"`
+	AuthenticationMechanism       string            `help:"Authentication mechanism. Default is empty. Use MONGODB-X509 for ssl certificates"`
+
+	flags.LogLevelFatalFlags
+}
+
+// RunCmd executes the AddAgentQANMongoDBMongologAgentCommand and returns the result.
+func (cmd *AddAgentQANMongoDBMongologAgentCommand) RunCmd() (commands.Result, error) {
+	customLabels := commands.ParseCustomLabels(cmd.CustomLabels)
+
+	tlsCertificateKey, err := commands.ReadFile(cmd.TLSCertificateKeyFile)
+	if err != nil {
+		return nil, err
+	}
+	tlsCa, err := commands.ReadFile(cmd.TLSCaFile)
+	if err != nil {
+		return nil, err
+	}
+
+	params := &agents.AddAgentParams{
+		Body: agents.AddAgentBody{
+			QANMongodbMongologAgent: &agents.AddAgentParamsBodyQANMongodbMongologAgent{
+				PMMAgentID:                    cmd.PMMAgentID,
+				ServiceID:                     cmd.ServiceID,
+				Username:                      cmd.Username,
+				Password:                      cmd.Password,
+				CustomLabels:                  customLabels,
+				SkipConnectionCheck:           cmd.SkipConnectionCheck,
+				MaxQueryLength:                cmd.MaxQueryLength,
+				TLS:                           cmd.TLS,
+				TLSSkipVerify:                 cmd.TLSSkipVerify,
+				TLSCertificateKey:             tlsCertificateKey,
+				TLSCertificateKeyFilePassword: cmd.TLSCertificateKeyFilePassword,
+				TLSCa:                         tlsCa,
+				AuthenticationMechanism:       cmd.AuthenticationMechanism,
+				LogLevel:                      cmd.LogLevelFatalFlags.LogLevel.EnumValue(),
+			},
+		},
+		Context: commands.Ctx,
+	}
+
+	resp, err := client.Default.AgentsService.AddAgent(params)
+	if err != nil {
+		return nil, err
+	}
+	return &addAgentQANMongoDBMongologAgentResult{
+		Agent: resp.Payload.QANMongodbMongologAgent,
+	}, nil
+}

admin/commands/inventory/inventory.go

@@ -55,6 +55,7 @@ type AddAgentCommand struct {
 	ProxysqlExporter AddAgentProxysqlExporterCommand `cmd:"" help:"Add proxysql_exporter to inventory"`
 
 	QANMongoDBProfilerAgent         AddAgentQANMongoDBProfilerAgentCommand         `cmd:"" name:"qan-mongodb-profiler-agent" help:"Add QAN MongoDB profiler agent to inventory"`
+	QANMongoDBMongologAgent         AddAgentQANMongoDBMongologAgentCommand         `cmd:"" name:"qan-mongodb-mongolog-agent" help:"Add QAN MongoDB mongolog agent to inventory"`
 	QANMySQLPerfSchemaAgent         AddAgentQANMySQLPerfSchemaAgentCommand         `cmd:"" name:"qan-mysql-perfschema-agent" help:"Add QAN MySQL perf schema agent to inventory"`
 	QANMySQLSlowlogAgent            AddAgentQANMySQLSlowlogAgentCommand            `cmd:"" name:"qan-mysql-slowlog-agent" help:"Add QAN MySQL slowlog agent to inventory"`
 	QANPostgreSQLPgStatementsAgent  AddAgentQANPostgreSQLPgStatementsAgentCommand  `cmd:"" name:"qan-postgresql-pgstatements-agent" help:"Add QAN PostgreSQL Stat Statements Agent to inventory"`

admin/commands/inventory/list_agents.go

@@ -47,6 +47,7 @@ var acceptableAgentTypes = map[string][]string{
 	types.AgentTypeQANMySQLPerfSchemaAgent:         {types.AgentTypeName(types.AgentTypeQANMySQLPerfSchemaAgent), "qan-mysql-perfschema-agent"},
 	types.AgentTypeQANMySQLSlowlogAgent:            {types.AgentTypeName(types.AgentTypeQANMySQLSlowlogAgent), "qan-mysql-slowlog-agent"},
 	types.AgentTypeQANMongoDBProfilerAgent:         {types.AgentTypeName(types.AgentTypeQANMongoDBProfilerAgent), "qan-mongodb-profiler-agent"},
+	types.AgentTypeQANMongoDBMongologAgent:         {types.AgentTypeName(types.AgentTypeQANMongoDBMongologAgent), "qan-mongodb-mongolog-agent"},
 	types.AgentTypeQANPostgreSQLPgStatementsAgent:  {types.AgentTypeName(types.AgentTypeQANPostgreSQLPgStatementsAgent), "qan-postgresql-pgstatements-agent"},
 	types.AgentTypeQANPostgreSQLPgStatMonitorAgent: {types.AgentTypeName(types.AgentTypeQANPostgreSQLPgStatMonitorAgent), "qan-postgresql-pgstatmonitor-agent"},
 	types.AgentTypeRDSExporter:                     {types.AgentTypeName(types.AgentTypeRDSExporter), "rds-exporter"},

admin/commands/list.go

@@ -286,6 +286,7 @@ func agentsList(agentsRes *agents.ListAgentsOK, nodeID string) []listResultAgent
 	agentsList = append(agentsList, qanMysqlPerfschemaAgents(agentsRes, pmmAgentIDs)...)
 	agentsList = append(agentsList, qanMysqlSlowlogAgents(agentsRes, pmmAgentIDs)...)
 	agentsList = append(agentsList, qanMongodbProfilerAgents(agentsRes, pmmAgentIDs)...)
+	agentsList = append(agentsList, qanMongodbMongologAgents(agentsRes, pmmAgentIDs)...)
 	agentsList = append(agentsList, qanPostgresqlPgstatementsAgents(agentsRes, pmmAgentIDs)...)
 	agentsList = append(agentsList, qanPostgresqlPgstatmonitorAgents(agentsRes, pmmAgentIDs)...)
 	agentsList = append(agentsList, externalExporters(agentsRes, nodeID)...)
@@ -434,6 +435,22 @@ func qanMongodbProfilerAgents(agentsRes *agents.ListAgentsOK, pmmAgentIDs map[st
 	return agentsList
 }
 
+func qanMongodbMongologAgents(agentsRes *agents.ListAgentsOK, pmmAgentIDs map[string]struct{}) []listResultAgent {
+	var agentsList []listResultAgent
+	for _, a := range agentsRes.Payload.QANMongodbMongologAgent {
+		if _, ok := pmmAgentIDs[a.PMMAgentID]; ok {
+			agentsList = append(agentsList, listResultAgent{
+				AgentType: types.AgentTypeQANMongoDBMongologAgent,
+				AgentID:   a.AgentID,
+				ServiceID: a.ServiceID,
+				Status:    getStatus(a.Status),
+				Disabled:  a.Disabled,
+			})
+		}
+	}
+	return agentsList
+}
+
 func qanPostgresqlPgstatementsAgents(agentsRes *agents.ListAgentsOK, pmmAgentIDs map[string]struct{}) []listResultAgent {
 	var agentsList []listResultAgent
 	for _, a := range agentsRes.Payload.QANPostgresqlPgstatementsAgent {

admin/commands/list_test.go

@@ -714,6 +714,32 @@ func TestQanMongodbProfilerAgents(t *testing.T) {
 	assert.Equal(t, types.AgentTypeQANMongoDBProfilerAgent, result[0].AgentType)
 }
 
+func TestQanMongodbMongologAgents(t *testing.T) {
+	t.Parallel()
+
+	pmmAgentIDs := map[string]struct{}{
+		"pmm-agent-1": {},
+	}
+
+	agentsRes := &agents_service.ListAgentsOK{
+		Payload: &agents_service.ListAgentsOKBody{
+			QANMongodbMongologAgent: []*agents_service.ListAgentsOKBodyQANMongodbMongologAgentItems0{
+				{
+					AgentID:    "qan-mongodb-monoglog-1",
+					PMMAgentID: "pmm-agent-1",
+					ServiceID:  "mongodb-service-1",
+					Status:     pointer.ToString("AGENT_STATUS_RUNNING"),
+				},
+			},
+		},
+	}
+
+	result := qanMongodbMongologAgents(agentsRes, pmmAgentIDs)
+
+	assert.Len(t, result, 1)
+	assert.Equal(t, types.AgentTypeQANMongoDBMongologAgent, result[0].AgentType)
+}
+
 func TestQanPostgresqlPgstatementsAgents(t *testing.T) {
 	t.Parallel()
 

admin/commands/management/add_mongodb.go

@@ -27,6 +27,8 @@ import (
 const (
 	// MongodbQuerySourceProfiler defines available source name for profiler.
 	MongodbQuerySourceProfiler = "profiler"
+	// MongodbQuerySourceMongolog defines available source name for profiler.
+	MongodbQuerySourceMongolog = "mongolog"
 	// MongodbQuerySourceNone defines available source name for profiler.
 	MongodbQuerySourceNone = "none"
 )
@@ -175,6 +177,7 @@ func (cmd *AddMongoDBCommand) RunCmd() (commands.Result, error) {
 				AgentPassword:  cmd.AgentPassword,
 
 				QANMongodbProfiler: cmd.QuerySource == MongodbQuerySourceProfiler,
+				QANMongodbMongolog: cmd.QuerySource == MongodbQuerySourceMongolog,
 
 				CustomLabels:                  customLabels,
 				SkipConnectionCheck:           cmd.SkipConnectionCheck,

agent/Makefile

@@ -105,9 +105,12 @@ _run:
 ENV_UP_FLAGS ?= --force-recreate --renew-anon-volumes --remove-orphans
 
 env-up:                         ## Start development environment
-	# to make slowlog rotation tests work
 	rm -fr testdata
+	# to make slowlog rotation tests work
 	mkdir -p testdata/mysql/slowlogs
+	# to make mongolog tests work
+	mkdir -p testdata/mongo/var/log/mongodb
+	touch testdata/mongo/var/log/mongodb/mongo.log testdata/mongo/var/log/mongodb/mongo_tls.log
 	chmod -R 0777 testdata
 
 	docker compose up $(ENV_UP_FLAGS)