Skip to content

Commit d53b071

Browse files
aetaricshanehughes1990
aetaric
and
shanehughes1990
authored
3.2.0: Splunk HEC stats (metrics) and notifications (events), bug fixes (#73)
* WIP: fixes certain problems with baseurl not being '/' * fix webserver formatting * update go deps * adds splunk HEC for stats * add Splunk HTTP Event Collector to notifications * add example blocks for Splunk HEC in stats and notifications * adding baseURL type and enforcing trailing slash FromConfig (#71) * adding baseURL type and enforcing trailing slash FromConfig * fix api routes in react code. fix minor issue in webserver.go for embeded fs pathing. --------- Co-authored-by: Dustin Essington <[email protected]> * make Splunk HEC calls async to improve execution time --------- Co-authored-by: Shane Hughes <[email protected]>
1 parent 85747c9 commit d53b071

File tree

13 files changed

+657
-217
lines changed

13 files changed

+657
-217
lines changed

‎checkrr.yaml.example

+11
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,14 @@ notifications:
120120
- unknowndetected
121121
- startrun
122122
- endrun
123+
splunk:
124+
url: ""
125+
token: ""
126+
notificationtypes:
127+
- reacquire
128+
- unknowndetected
129+
- startrun
130+
- endrun
123131
stats: # These will slow down the runtime substantually, but... DATA
124132
influxdb1:
125133
url: ""
@@ -131,6 +139,9 @@ stats: # These will slow down the runtime substantually, but... DATA
131139
token: ""
132140
org: "default"
133141
bucket: "checkrr"
142+
splunk:
143+
address: "https://127.0.0.1:8088/services/collector" # You must have HTTP Event Collection enabled and configured
144+
token: "HEC Token" # this is the HEC token for your input. Configure it to force ingestion into a metrics index
134145
webserver:
135146
port: 8585
136147
baseurl: "/"

‎features/stats.go

+61-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"net/http"
78
"os"
9+
"strings"
810
"time"
911

1012
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
@@ -22,6 +24,8 @@ type Stats struct {
2224
writeAPI2 api.WriteAPIBlocking `json:"-"`
2325
config viper.Viper `json:"-"`
2426
Log log.Logger `json:"-"`
27+
splunk Splunk `json:"-"`
28+
splunkConfigured bool `json:"-"`
2529
SonarrSubmissions uint64 `json:"sonarrSubmissions"`
2630
RadarrSubmissions uint64 `json:"radarrSubmissions"`
2731
LidarrSubmissions uint64 `json:"lidarrSubmissions"`
@@ -39,6 +43,30 @@ type Stats struct {
3943
DB *bolt.DB `json:"-"`
4044
}
4145

46+
type SplunkStats struct {
47+
Fields *SplunkFields `json:"fields"`
48+
Time int64 `json:"time"`
49+
Event string `json:"event"`
50+
}
51+
52+
type SplunkFields struct {
53+
SonarrSubmissions uint64 `json:"metric_name:checkrr.sonarrSubmissions"`
54+
RadarrSubmissions uint64 `json:"metric_name:checkrr.radarrSubmissions"`
55+
LidarrSubmissions uint64 `json:"metric_name:checkrr.lidarrSubmissions"`
56+
FilesChecked uint64 `json:"metric_name:checkrr.filesChecked"`
57+
HashMatches uint64 `json:"metric_name:checkrr.hashMatches"`
58+
HashMismatches uint64 `json:"metric_name:checkrr.hashMismatches"`
59+
VideoFiles uint64 `json:"metric_name:checkrr.videoFiles"`
60+
AudioFiles uint64 `json:"metric_name:checkrr.audioFiles"`
61+
UnknownFileCount uint64 `json:"metric_name:checkrr.unknownFileCount"`
62+
NonVideo uint64 `json:"metric_name:checkrr.nonVideo"`
63+
}
64+
65+
type Splunk struct {
66+
address string
67+
token string
68+
}
69+
4270
func (s *Stats) FromConfig(config viper.Viper) {
4371
s.config = config
4472
if config.Sub("influxdb1") != nil {
@@ -63,6 +91,12 @@ func (s *Stats) FromConfig(config viper.Viper) {
6391
s.writeAPI2.EnableBatching()
6492
s.Log.WithFields(log.Fields{"startup": true, "influxdb": "enabled"}).Info("Sending data to InfluxDB 2.x")
6593
}
94+
if config.Sub("splunk") != nil {
95+
splunk := config.Sub("splunk")
96+
s.splunk = Splunk{address: splunk.GetString("address"), token: splunk.GetString("token")}
97+
s.splunkConfigured = true
98+
s.Log.WithFields(log.Fields{"startup": true, "splunk stats": "enabled"}).Info("Sending stats data to Splunk")
99+
}
66100
}
67101

68102
func (s *Stats) Start() {
@@ -134,7 +168,7 @@ func (s *Stats) Render() {
134168
t.Render()
135169
}
136170

137-
func (s Stats) Write(field string, count uint64) {
171+
func (s *Stats) Write(field string, count uint64) {
138172
// Send to influxdb if enabled
139173
if s.writeAPI1 != nil {
140174
p := influxdb2.NewPointWithMeasurement("checkrr").
@@ -151,6 +185,32 @@ func (s Stats) Write(field string, count uint64) {
151185
SetTime(time.Now())
152186
s.writeAPI2.WritePoint(context.Background(), p)
153187
}
188+
// Send to splunk if configured
189+
if s.splunkConfigured {
190+
t := time.Now().Unix()
191+
splunkfields := SplunkFields{FilesChecked: s.FilesChecked, HashMatches: s.HashMatches, HashMismatches: s.HashMismatches,
192+
SonarrSubmissions: s.SonarrSubmissions, RadarrSubmissions: s.RadarrSubmissions, LidarrSubmissions: s.LidarrSubmissions,
193+
VideoFiles: s.VideoFiles, NonVideo: s.NonVideo, AudioFiles: s.AudioFiles, UnknownFileCount: s.UnknownFileCount}
194+
splunkstats := SplunkStats{Event: "metric", Time: t, Fields: &splunkfields}
195+
go func(splunkstats SplunkStats) {
196+
client := &http.Client{}
197+
j, _ := json.Marshal(splunkstats)
198+
var data = strings.NewReader(string(j))
199+
req, err := http.NewRequest("POST", s.splunk.address, data)
200+
if err != nil {
201+
log.Warn(err)
202+
}
203+
req.Header.Set("Authorization", fmt.Sprintf("Splunk %s", s.splunk.token))
204+
resp, err := client.Do(req)
205+
if err != nil {
206+
log.Warn(err)
207+
}
208+
if resp.StatusCode != 200 {
209+
log.Warnf("Recieved %d status code from Splunk", resp.StatusCode)
210+
}
211+
defer resp.Body.Close()
212+
}(splunkstats)
213+
}
154214
// Update stats DB
155215
err := s.DB.Update(func(tx *bolt.Tx) error {
156216
b := tx.Bucket([]byte("Checkrr-stats"))

‎go.mod

+64-34
Original file line numberDiff line numberDiff line change
@@ -4,51 +4,47 @@ go 1.19
44

55
require (
66
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
7-
github.com/disgoorg/disgo v0.16.7
7+
github.com/disgoorg/disgo v0.16.11
88
github.com/disgoorg/snowflake/v2 v2.0.1
99
github.com/gin-contrib/static v0.0.1
1010
github.com/gin-gonic/gin v1.9.1
1111
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
12-
github.com/gregdel/pushover v1.2.0
12+
github.com/gregdel/pushover v1.3.0
1313
github.com/h2non/filetype v1.1.3
1414
github.com/influxdata/influxdb-client-go/v2 v2.12.3
15-
github.com/jedib0t/go-pretty/v6 v6.4.6
15+
github.com/jedib0t/go-pretty/v6 v6.4.7
1616
github.com/kalafut/imohash v1.0.2
1717
github.com/robfig/cron/v3 v3.0.1
1818
github.com/sirupsen/logrus v1.9.3
1919
github.com/spf13/pflag v1.0.5
2020
github.com/spf13/viper v1.16.0
2121
github.com/xconstruct/go-pushbullet v0.0.0-20171206132031-67759df45fbb
2222
go.etcd.io/bbolt v1.3.7
23-
golift.io/starr v0.14.1-0.20221014065949-c78a10375d37
23+
golift.io/starr v1.0.0
2424
gopkg.in/vansante/go-ffprobe.v2 v2.1.1
2525
)
2626

2727
require (
28-
github.com/bytedance/sonic v1.10.0-rc // indirect
29-
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
30-
github.com/chenzhuoyu/iasm v0.9.0 // indirect
31-
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
32-
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
33-
github.com/labstack/echo/v4 v4.11.1 // indirect
34-
github.com/labstack/gommon v0.4.0 // indirect
35-
github.com/mattn/go-colorable v0.1.13 // indirect
36-
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
37-
github.com/valyala/bytebufferpool v1.0.0 // indirect
38-
github.com/valyala/fasttemplate v1.2.2 // indirect
39-
go.opentelemetry.io/otel/metric v1.16.0 // indirect
40-
golang.org/x/arch v0.4.0 // indirect
41-
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
42-
)
43-
44-
require (
28+
github.com/BurntSushi/toml v1.3.2 // indirect
29+
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect
30+
github.com/CloudyKit/jet/v6 v6.2.0 // indirect
31+
github.com/Joker/jade v1.1.3 // indirect
32+
github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect
33+
github.com/andybalholm/brotli v1.0.5 // indirect
4534
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
4635
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
36+
github.com/aymerick/douceur v0.2.0 // indirect
37+
github.com/bytedance/sonic v1.10.1 // indirect
38+
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
39+
github.com/chenzhuoyu/iasm v0.9.0 // indirect
4740
github.com/davecgh/go-spew v1.1.1 // indirect
48-
github.com/deepmap/oapi-codegen v1.13.0 // indirect
41+
github.com/deepmap/oapi-codegen v1.15.0 // indirect
4942
github.com/disgoorg/json v1.1.0 // indirect
5043
github.com/disgoorg/log v1.2.1 // indirect
44+
github.com/fatih/structs v1.1.0 // indirect
45+
github.com/flosch/pongo2/v4 v4.0.2 // indirect
5146
github.com/fsnotify/fsnotify v1.6.0 // indirect
47+
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
5248
github.com/gin-contrib/sse v0.1.0 // indirect
5349
github.com/go-logr/logr v1.2.4 // indirect
5450
github.com/go-logr/stdr v1.2.2 // indirect
@@ -64,43 +60,77 @@ require (
6460
github.com/go-openapi/validate v0.22.1 // indirect
6561
github.com/go-playground/locales v0.14.1 // indirect
6662
github.com/go-playground/universal-translator v0.18.1 // indirect
67-
github.com/go-playground/validator/v10 v10.14.1 // indirect
63+
github.com/go-playground/validator/v10 v10.15.5 // indirect
6864
github.com/goccy/go-json v0.10.2 // indirect
69-
github.com/google/uuid v1.3.0 // indirect
65+
github.com/golang/snappy v0.0.4 // indirect
66+
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect
67+
github.com/google/uuid v1.3.1 // indirect
68+
github.com/gorilla/css v1.0.0 // indirect
69+
github.com/gorilla/websocket v1.5.0 // indirect
7070
github.com/gotify/go-api-client/v2 v2.0.4
7171
github.com/hashicorp/hcl v1.0.0 // indirect
7272
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
73+
github.com/iris-contrib/schema v0.0.6 // indirect
7374
github.com/josharian/intern v1.0.0 // indirect
7475
github.com/json-iterator/go v1.1.12 // indirect
76+
github.com/kataras/blocks v0.0.8 // indirect
77+
github.com/kataras/golog v0.1.9 // indirect
78+
github.com/kataras/iris/v12 v12.2.7 // indirect
79+
github.com/kataras/pio v0.0.12 // indirect
80+
github.com/kataras/sitemap v0.0.6 // indirect
81+
github.com/kataras/tunnel v0.0.4 // indirect
82+
github.com/klauspost/compress v1.17.0 // indirect
83+
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
84+
github.com/knz/go-libedit v1.10.1 // indirect
85+
github.com/labstack/echo/v4 v4.11.1 // indirect
86+
github.com/labstack/gommon v0.4.0 // indirect
7587
github.com/leodido/go-urn v1.2.4 // indirect
7688
github.com/magiconair/properties v1.8.7 // indirect
89+
github.com/mailgun/raymond/v2 v2.0.48 // indirect
7790
github.com/mailru/easyjson v0.7.7 // indirect
91+
github.com/mattn/go-colorable v0.1.13 // indirect
7892
github.com/mattn/go-isatty v0.0.19 // indirect
7993
github.com/mattn/go-runewidth v0.0.15 // indirect
94+
github.com/microcosm-cc/bluemonday v1.0.25 // indirect
8095
github.com/mitchellh/mapstructure v1.5.0 // indirect
8196
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
8297
github.com/modern-go/reflect2 v1.0.2 // indirect
8398
github.com/oklog/ulid v1.3.1 // indirect
8499
github.com/opentracing/opentracing-go v1.2.0 // indirect
85-
github.com/pelletier/go-toml/v2 v2.0.9 // indirect
100+
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
86101
github.com/pkg/errors v0.9.1 // indirect
87102
github.com/pmezard/go-difflib v1.0.0 // indirect
88103
github.com/rivo/uniseg v0.4.4 // indirect
104+
github.com/russross/blackfriday/v2 v2.1.0 // indirect
89105
github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b // indirect
90-
github.com/spf13/afero v1.9.5 // indirect
106+
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
107+
github.com/spf13/afero v1.10.0 // indirect
91108
github.com/spf13/cast v1.5.1 // indirect
92109
github.com/spf13/jwalterweatherman v1.1.0 // indirect
110+
github.com/stretchr/objx v0.5.0 // indirect
93111
github.com/stretchr/testify v1.8.4 // indirect
94-
github.com/subosito/gotenv v1.4.2 // indirect
112+
github.com/subosito/gotenv v1.6.0 // indirect
113+
github.com/tdewolff/minify/v2 v2.12.9 // indirect
114+
github.com/tdewolff/parse/v2 v2.6.8 // indirect
115+
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
95116
github.com/twmb/murmur3 v1.1.8 // indirect
96117
github.com/ugorji/go/codec v1.2.11 // indirect
97-
go.mongodb.org/mongo-driver v1.12.0 // indirect
98-
go.opentelemetry.io/otel v1.16.0 // indirect
99-
go.opentelemetry.io/otel/trace v1.16.0 // indirect
100-
golang.org/x/crypto v0.11.0 // indirect
101-
golang.org/x/net v0.12.0 // indirect
102-
golang.org/x/sys v0.10.0 // indirect
103-
golang.org/x/text v0.11.0 // indirect
118+
github.com/valyala/bytebufferpool v1.0.0 // indirect
119+
github.com/valyala/fasttemplate v1.2.2 // indirect
120+
github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect
121+
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
122+
github.com/yosssi/ace v0.0.5 // indirect
123+
go.mongodb.org/mongo-driver v1.12.1 // indirect
124+
go.opentelemetry.io/otel v1.19.0 // indirect
125+
go.opentelemetry.io/otel/metric v1.19.0 // indirect
126+
go.opentelemetry.io/otel/trace v1.19.0 // indirect
127+
golang.org/x/arch v0.5.0 // indirect
128+
golang.org/x/crypto v0.13.0 // indirect
129+
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
130+
golang.org/x/net v0.15.0 // indirect
131+
golang.org/x/sys v0.12.0 // indirect
132+
golang.org/x/text v0.13.0 // indirect
133+
golang.org/x/time v0.3.0 // indirect
104134
google.golang.org/protobuf v1.31.0 // indirect
105135
gopkg.in/ini.v1 v1.67.0 // indirect
106136
gopkg.in/yaml.v2 v2.4.0 // indirect

0 commit comments

Comments
 (0)