From 6e49623870b0e01550b67fab14fd1ef4901e4263 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Mon, 19 Dec 2022 21:14:10 +0530 Subject: [PATCH 1/6] Support user databases for transaction log --- .../_dev/build/docs/README.md | 3 ++- packages/microsoft_sqlserver/changelog.yml | 5 +++++ .../agent/stream/stream.yml.hbs | 18 +++++++++++------- .../data_stream/transaction_log/manifest.yml | 2 +- packages/microsoft_sqlserver/docs/README.md | 3 ++- packages/microsoft_sqlserver/manifest.yml | 2 +- 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md index 4c997427a0a..4a1aa102250 100644 --- a/packages/microsoft_sqlserver/_dev/build/docs/README.md +++ b/packages/microsoft_sqlserver/_dev/build/docs/README.md @@ -75,7 +75,8 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects system level `transaction_log` metrics information for SQL Server instance. +Collects `transaction_log` metrics information for all the system and user Dbs of SQL Server instance. +The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index d0043ba8056..b2e2e83c64e 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: Support user databases for transaction log. + type: enhancement + link: tbd - version: "1.10.0" changes: - description: Ingest instance name and server names. diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs index ef2cdc5a142..3424200103f 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs @@ -5,15 +5,19 @@ hosts: period: {{period}} driver: mssql raw_data.enabled: true +databases: +{{#each databases}} + - {{this}} +{{/each}} # Collect the transaction logs from the system database sql_queries: + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases;" + response_format: table + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', s.database_id, total_log_size_mb, active_log_size_mb,log_backup_time,log_since_last_log_backup_mb,log_since_last_checkpoint_mb,log_recovery_size_mb from sys.databases As s CROSS APPLY sys.dm_db_log_stats(s.database_id);" + response_format: table {{#if databases}} {{#each databases as |database_name i|}} - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table + - query: "USE {{database_name}} ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table {{/each}} -{{/if}} +{{/if}} \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml index a4705806094..5a7040dc935 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml @@ -17,7 +17,7 @@ streams: title: Databases multi: true required: true - show_user: false + show_user: true description: Default system databases are preloaded. For custom database please add additional rows and enter the database name. default: ["master", "model", "tempdb", "msdb"] title: Microsoft SQL Server transaction_log metrics diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index 8fbc61282e3..0e2d8b98d86 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -75,7 +75,8 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects system level `transaction_log` metrics information for SQL Server instance. +Collects `transaction_log` metrics information for all the system and user Dbs of SQL Server instance. +The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml index 08a1242b70f..05d253f3eda 100644 --- a/packages/microsoft_sqlserver/manifest.yml +++ b/packages/microsoft_sqlserver/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: microsoft_sqlserver title: "Microsoft SQL Server" -version: "1.10.0" +version: "1.11.0" license: basic description: Collect events from Microsoft SQL Server with Elastic Agent type: integration From 04cd8a30c200543502bdd381e7da8a88a5b86b11 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Mon, 19 Dec 2022 21:25:35 +0530 Subject: [PATCH 2/6] Update changelog.yml --- packages/microsoft_sqlserver/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index b2e2e83c64e..f489356765f 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Support user databases for transaction log. type: enhancement - link: tbd + link: https://github.com/elastic/integrations/pull/4869 - version: "1.10.0" changes: - description: Ingest instance name and server names. From 61afa21f840c01aa605e6ebb2d4ee4093b94f989 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Tue, 20 Dec 2022 13:15:34 +0530 Subject: [PATCH 3/6] Update sample_event.json --- .../transaction_log/sample_event.json | 73 ++++++++----------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json b/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json index 6c91b5516e2..b8aa2143cc4 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json @@ -1,67 +1,52 @@ { - "@timestamp": "2022-11-23T10:53:05.080Z", + "@timestamp": "2022-12-20T07:34:29.687Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", + "name": "192.168.1.2", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.5.3" }, "data_stream": { "dataset": "microsoft_sqlserver.transaction_log", - "namespace": "ep", + "namespace": "default", "type": "metrics" }, "ecs": { "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", "snapshot": false, - "version": "8.5.0" + "version": "8.5.3" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.transaction_log", - "duration": 1449739900, - "ingested": "2022-11-23T10:53:07Z", + "duration": 2147044750, + "ingested": "2022-12-20T07:34:32Z", "module": "sql" }, "host": { "architecture": "x86_64", "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "hostname": "192.168.1.2", + "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875", "ip": [ - "172.18.0.5" + "192.168.1.2" ], "mac": [ - "02-42-AC-12-00-05" + "36-F7-DC-28-23-80" ], - "name": "docker-fleet-agent", + "name": "192.168.1.2", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "21D62", + "family": "darwin", + "kernel": "21.3.0", + "name": "macOS", + "platform": "darwin", + "type": "macos", + "version": "12.2.1" } }, "metricset": { @@ -70,18 +55,18 @@ }, "mssql": { "metrics": { - "database_id": 1, - "database_name": "master", + "server_name": "obs-int-mssql20", "instance_name": "MSSQLSERVER", - "log_space_in_bytes_since_last_backup": 139264, - "server_name": "obs-service-int", + "database_name": "master", + "database_id": 1, + "log_space_in_bytes_since_last_backup": 602112, "total_log_size_bytes": 2088960, - "used_log_space_bytes": 626688, - "used_log_space_pct": 30 + "used_log_space_pct": 49.01960754394531, + "used_log_space_bytes": 1024000 } }, "service": { - "address": "20.121.218.233", + "address": "20.228.135.242", "type": "sql" } } \ No newline at end of file From 386e0ce3c5fb856fd3f82f392d74f884efdf6220 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Tue, 20 Dec 2022 13:27:42 +0530 Subject: [PATCH 4/6] Update README.md --- packages/microsoft_sqlserver/docs/README.md | 73 ++++++++------------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index 0e2d8b98d86..da164a74aac 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -495,69 +495,54 @@ An example event for `transaction_log` looks as following: ```json { - "@timestamp": "2022-11-23T10:53:05.080Z", + "@timestamp": "2022-12-20T07:34:29.687Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", + "name": "192.168.1.2", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.5.3" }, "data_stream": { "dataset": "microsoft_sqlserver.transaction_log", - "namespace": "ep", + "namespace": "default", "type": "metrics" }, "ecs": { "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", "snapshot": false, - "version": "8.5.0" + "version": "8.5.3" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.transaction_log", - "duration": 1449739900, - "ingested": "2022-11-23T10:53:07Z", + "duration": 2147044750, + "ingested": "2022-12-20T07:34:32Z", "module": "sql" }, "host": { "architecture": "x86_64", "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "hostname": "192.168.1.2", + "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875", "ip": [ - "172.18.0.5" + "192.168.1.2" ], "mac": [ - "02-42-AC-12-00-05" + "36-F7-DC-28-23-80" ], - "name": "docker-fleet-agent", + "name": "192.168.1.2", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "21D62", + "family": "darwin", + "kernel": "21.3.0", + "name": "macOS", + "platform": "darwin", + "type": "macos", + "version": "12.2.1" } }, "metricset": { @@ -566,18 +551,18 @@ An example event for `transaction_log` looks as following: }, "mssql": { "metrics": { - "database_id": 1, - "database_name": "master", + "server_name": "obs-int-mssql20", "instance_name": "MSSQLSERVER", - "log_space_in_bytes_since_last_backup": 139264, - "server_name": "obs-service-int", + "database_name": "master", + "database_id": 1, + "log_space_in_bytes_since_last_backup": 602112, "total_log_size_bytes": 2088960, - "used_log_space_bytes": 626688, - "used_log_space_pct": 30 + "used_log_space_pct": 49.01960754394531, + "used_log_space_bytes": 1024000 } }, "service": { - "address": "20.121.218.233", + "address": "20.228.135.242", "type": "sql" } } From 50ce9003e65dcbee224d14a3fed7c871c83ab815 Mon Sep 17 00:00:00 2001 From: ritalwar Date: Wed, 21 Dec 2022 09:54:15 +0530 Subject: [PATCH 5/6] Address review comments --- packages/microsoft_sqlserver/_dev/build/docs/README.md | 2 +- .../data_stream/transaction_log/manifest.yml | 2 +- packages/microsoft_sqlserver/docs/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md index 4a1aa102250..d3ac3204ec9 100644 --- a/packages/microsoft_sqlserver/_dev/build/docs/README.md +++ b/packages/microsoft_sqlserver/_dev/build/docs/README.md @@ -75,7 +75,7 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects `transaction_log` metrics information for all the system and user Dbs of SQL Server instance. +Collects `transaction_log` metrics information for all the system and user dbs of SQL Server instance. The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml index 5a7040dc935..1a13dca435e 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml @@ -18,7 +18,7 @@ streams: multi: true required: true show_user: true - description: Default system databases are preloaded. For custom database please add additional rows and enter the database name. + description: Default system databases are preloaded. For user defined databases please add additional rows and enter the database name. default: ["master", "model", "tempdb", "msdb"] title: Microsoft SQL Server transaction_log metrics description: Collect Microsoft SQL Server transaction_log metrics diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index da164a74aac..c7c08a1f7ee 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -75,7 +75,7 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects `transaction_log` metrics information for all the system and user Dbs of SQL Server instance. +Collects `transaction_log` metrics information for all the system and user dbs of SQL Server instance. The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) From 3d9368a166c397729400e81f2c79ab634506448e Mon Sep 17 00:00:00 2001 From: ritalwar Date: Thu, 22 Dec 2022 17:17:46 +0530 Subject: [PATCH 6/6] Chages for short term solution --- .../microsoft_sqlserver/_dev/build/docs/README.md | 4 ++-- .../transaction_log/agent/stream/stream.yml.hbs | 12 ++++-------- packages/microsoft_sqlserver/docs/README.md | 4 ++-- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md index d3ac3204ec9..2decfd86e9d 100644 --- a/packages/microsoft_sqlserver/_dev/build/docs/README.md +++ b/packages/microsoft_sqlserver/_dev/build/docs/README.md @@ -75,8 +75,8 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects `transaction_log` metrics information for all the system and user dbs of SQL Server instance. -The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. +Collects system level `transaction_log` metrics information for SQL Server instance. +Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs index 3424200103f..99982ef7b00 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs @@ -5,18 +5,14 @@ hosts: period: {{period}} driver: mssql raw_data.enabled: true -databases: -{{#each databases}} - - {{this}} -{{/each}} # Collect the transaction logs from the system database sql_queries: - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases;" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', s.database_id, total_log_size_mb, active_log_size_mb,log_backup_time,log_since_last_log_backup_mb,log_since_last_checkpoint_mb,log_recovery_size_mb from sys.databases As s CROSS APPLY sys.dm_db_log_stats(s.database_id);" - response_format: table {{#if databases}} {{#each databases as |database_name i|}} + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" + response_format: table + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table - query: "USE {{database_name}} ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" response_format: table {{/each}} diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index c7c08a1f7ee..e9956a82648 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -75,8 +75,8 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics -Collects `transaction_log` metrics information for all the system and user dbs of SQL Server instance. -The metrics from sys.dm_db_log_space_usage table for user dbs can be collected by providing list of user databases, by default it will collect metrics for system dbs. +Collects system level `transaction_log` metrics information for SQL Server instance. +Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15)