Skip to content

Commit 1a0dbf6

Browse files
alexluongalexluong
committed
feat: Telemetry setup
feat: Telemetry using buffered channel refactor: Telemetry config feat: Send telemetry event to Hookdeck feat: Sentry feat: Persist installation ID in Redis for telemetry chore: Only register dev route in debug mode refactor: Rename telemetry event fields chore: Populate application info build: Version
1 parent fd02610 commit 1a0dbf6

17 files changed

+458
-7
lines changed

‎build/.goreleaser.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ builds:
1111
- id: outpost
1212
ldflags:
1313
- -s -w
14+
- -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
15+
- -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
1416
binary: outpost
1517
env:
1618
- CGO_ENABLED=0
@@ -22,6 +24,8 @@ builds:
2224
- id: outpost-arm64
2325
ldflags:
2426
- -s -w
27+
- -X github.com/hookdeck/outpost/internal/version.version={{.Version}}
28+
- -X github.com/hookdeck/outpost/internal/version.commit={{.FullCommit}}
2529
binary: outpost
2630
env:
2731
- CGO_ENABLED=0

‎cmd/outpost/main.go

+7
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,17 @@ import (
77

88
"github.com/hookdeck/outpost/internal/app"
99
"github.com/hookdeck/outpost/internal/config"
10+
"github.com/hookdeck/outpost/internal/version"
1011
)
1112

1213
func main() {
1314
flags := config.ParseFlags()
15+
16+
if flags.Version {
17+
fmt.Println(version.Version())
18+
return
19+
}
20+
1421
cfg, err := config.Parse(flags)
1522
if err != nil {
1623
handleErr(err)

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ require (
1313
github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3
1414
github.com/aws/smithy-go v1.20.3
1515
github.com/caarlos0/env/v9 v9.0.0
16+
github.com/getsentry/sentry-go v0.31.1
17+
github.com/getsentry/sentry-go/gin v0.31.1
1618
github.com/gin-contrib/static v1.1.2
1719
github.com/gin-gonic/gin v1.10.0
1820
github.com/go-playground/validator/v10 v10.22.0

‎go.sum

+8
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,18 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
137137
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
138138
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
139139
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
140+
github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4=
141+
github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY=
142+
github.com/getsentry/sentry-go/gin v0.31.1 h1:lvOOO5j0o0IhYIXoHCmQ+D4ExhXWRCnDusV176dXWDA=
143+
github.com/getsentry/sentry-go/gin v0.31.1/go.mod h1:iMF6gA5uO2t3KVMj4QpjLi9B0U+oMidAiHAdPcJMMdQ=
140144
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
141145
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
142146
github.com/gin-contrib/static v1.1.2 h1:c3kT4bFkUJn2aoRU3s6XnMjJT8J6nNWJkR0NglqmlZ4=
143147
github.com/gin-contrib/static v1.1.2/go.mod h1:Fw90ozjHCmZBWbgrsqrDvO28YbhKEKzKp8GixhR4yLw=
144148
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
145149
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
150+
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
151+
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
146152
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
147153
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
148154
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
@@ -338,6 +344,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6
338344
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
339345
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
340346
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
347+
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
348+
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
341349
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
342350
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
343351
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

‎internal/app/app.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/hookdeck/outpost/internal/services/api"
1717
"github.com/hookdeck/outpost/internal/services/delivery"
1818
"github.com/hookdeck/outpost/internal/services/log"
19+
"github.com/hookdeck/outpost/internal/telemetry"
1920
"go.uber.org/zap"
2021
)
2122

@@ -59,6 +60,14 @@ func run(mainContext context.Context, cfg *config.Config) error {
5960
return err
6061
}
6162

63+
installationID, err := getInstallation(mainContext, cfg.Redis.ToConfig(), cfg.Telemetry.ToTelemetryConfig())
64+
if err != nil {
65+
return err
66+
}
67+
telemetry := telemetry.New(logger, cfg.Telemetry.ToTelemetryConfig(), installationID)
68+
telemetry.Init(mainContext)
69+
telemetry.ApplicationStarted(mainContext, cfg.ToTelemetryApplicationInfo())
70+
6271
// Set up cancellation context and waitgroup
6372
ctx, cancel := context.WithCancel(mainContext)
6473

@@ -86,6 +95,7 @@ func run(mainContext context.Context, cfg *config.Config) error {
8695
cfg,
8796
wg,
8897
logger,
98+
telemetry,
8999
)
90100
if err != nil {
91101
cancel()
@@ -109,6 +119,8 @@ func run(mainContext context.Context, cfg *config.Config) error {
109119
logger.Ctx(ctx).Info("context cancelled")
110120
}
111121

122+
telemetry.Flush()
123+
112124
// Handle shutdown
113125
cancel() // Signal cancellation to context.Context
114126
wg.Wait() // Block here until all workers are done
@@ -127,12 +139,13 @@ func constructServices(
127139
cfg *config.Config,
128140
wg *sync.WaitGroup,
129141
logger *logging.Logger,
142+
telemetry telemetry.Telemetry,
130143
) ([]Service, error) {
131144
serviceType := cfg.MustGetService()
132145
services := []Service{}
133146

134147
if serviceType == config.ServiceTypeAPI || serviceType == config.ServiceTypeAll {
135-
service, err := api.NewService(ctx, wg, cfg, logger)
148+
service, err := api.NewService(ctx, wg, cfg, logger, telemetry)
136149
if err != nil {
137150
return nil, err
138151
}

‎internal/app/installation.go

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package app
2+
3+
import (
4+
"context"
5+
6+
"github.com/google/uuid"
7+
"github.com/hookdeck/outpost/internal/redis"
8+
"github.com/hookdeck/outpost/internal/telemetry"
9+
)
10+
11+
const (
12+
outpostrcKey = "outpostrc"
13+
installationKey = "installation"
14+
)
15+
16+
func getInstallation(ctx context.Context, redisConfig *redis.RedisConfig, telemetryConfig telemetry.TelemetryConfig) (string, error) {
17+
if telemetryConfig.Disabled {
18+
return "", nil
19+
}
20+
21+
redisClient, err := redis.New(ctx, redisConfig)
22+
if err != nil {
23+
return "", err
24+
}
25+
26+
// TODO: consider using WATCH to avoid race condition
27+
// There's a potential race condition when multiple Outpost instances are started at the same time.
28+
// However, given this is for telemetry purposes, and it will be a temporary issue, we can ignore it for now.
29+
installationID, err := redisClient.HGet(ctx, outpostrcKey, installationKey).Result()
30+
if err != nil {
31+
if err == redis.Nil {
32+
installationID = uuid.New().String()
33+
if err = redisClient.HSet(ctx, outpostrcKey, installationKey, installationID).Err(); err != nil {
34+
return "", err
35+
}
36+
} else {
37+
return "", err
38+
}
39+
}
40+
41+
return installationID, nil
42+
}

‎internal/config/config.go

+52
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"github.com/hookdeck/outpost/internal/clickhouse"
1010
"github.com/hookdeck/outpost/internal/migrator"
1111
"github.com/hookdeck/outpost/internal/redis"
12+
"github.com/hookdeck/outpost/internal/telemetry"
13+
"github.com/hookdeck/outpost/internal/version"
1214
"github.com/joho/godotenv"
1315
"gopkg.in/yaml.v3"
1416
)
@@ -41,6 +43,7 @@ type Config struct {
4143
LogLevel string `yaml:"log_level" env:"LOG_LEVEL"`
4244
AuditLog bool `yaml:"audit_log" env:"AUDIT_LOG"`
4345
OpenTelemetry OpenTelemetryConfig `yaml:"otel"`
46+
Telemetry TelemetryConfig `yaml:"telemetry"`
4447

4548
// API
4649
APIPort int `yaml:"api_port" env:"API_PORT"`
@@ -150,6 +153,14 @@ func (c *Config) InitDefaults() {
150153
ConsecutiveFailureCount: 20,
151154
AutoDisableDestination: true,
152155
}
156+
157+
c.Telemetry = TelemetryConfig{
158+
Disabled: false,
159+
BatchSize: 100,
160+
BatchInterval: 5,
161+
HookdeckSourceURL: "https://hkdk.events/ih7t3hukydzlge",
162+
SentryDSN: "https://[email protected]/0",
163+
}
153164
}
154165

155166
func (c *Config) parseConfigFile(flagPath string, osInterface OSInterface) error {
@@ -325,6 +336,47 @@ func (c *Config) ConfigFilePath() string {
325336
return c.configPath
326337
}
327338

339+
type TelemetryConfig struct {
340+
Disabled bool `yaml:"disabled" env:"DISABLE_TELEMETRY"`
341+
BatchSize int `yaml:"batch_size" env:"TELEMETRY_BATCH_SIZE"`
342+
BatchInterval int `yaml:"batch_interval" env:"TELEMETRY_BATCH_INTERVAL"`
343+
HookdeckSourceURL string `yaml:"hookdeck_source_url" env:"TELEMETRY_HOOKDECK_SOURCE_URL"`
344+
SentryDSN string `yaml:"sentry_dsn" env:"TELEMETRY_SENTRY_DSN"`
345+
}
346+
347+
func (c *TelemetryConfig) ToTelemetryConfig() telemetry.TelemetryConfig {
348+
return telemetry.TelemetryConfig{
349+
Disabled: c.Disabled,
350+
BatchSize: c.BatchSize,
351+
BatchInterval: c.BatchInterval,
352+
HookdeckSourceURL: c.HookdeckSourceURL,
353+
SentryDSN: c.SentryDSN,
354+
}
355+
}
356+
357+
func (c *Config) ToTelemetryApplicationInfo() telemetry.ApplicationInfo {
358+
portalEnabled := c.APIKey != "" && c.APIJWTSecret != ""
359+
360+
entityStore := "redis"
361+
logStore := "TODO"
362+
if c.ClickHouse.Addr != "" {
363+
logStore = "clickhouse"
364+
}
365+
if c.PostgresURL != "" {
366+
logStore = "postgres"
367+
}
368+
369+
return telemetry.ApplicationInfo{
370+
Version: version.Version(),
371+
MQ: c.MQs.GetInfraType(),
372+
PortalEnabled: portalEnabled,
373+
EntityStore: entityStore,
374+
LogStore: logStore,
375+
}
376+
}
377+
378+
// ===== Misc =====
379+
328380
func (c *Config) ToMigratorOpts() migrator.MigrationOpts {
329381
return migrator.MigrationOpts{
330382
PG: migrator.MigrationOptsPG{

‎internal/config/flag.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,28 @@ package config
33
import "flag"
44

55
var (
6-
service string
7-
config string
6+
service string
7+
config string
8+
printVersion bool
89
)
910

1011
func init() {
1112
flag.StringVar(&service, "service", "", "service (e.g. api, delivery, log). If empty, all services will run.")
1213
flag.StringVar(&config, "config", "", "config file (e.g. .env, config.yaml)")
14+
flag.BoolVar(&printVersion, "version", false, "print version information")
1315
}
1416

1517
type Flags struct {
1618
Service string
1719
Config string // Config file path
20+
Version bool // Print version information
1821
}
1922

2023
func ParseFlags() Flags {
2124
flag.Parse()
2225
return Flags{
2326
Service: service,
2427
Config: config,
28+
Version: printVersion,
2529
}
2630
}

‎internal/services/api/api.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/hookdeck/outpost/internal/publishmq"
2121
"github.com/hookdeck/outpost/internal/redis"
2222
"github.com/hookdeck/outpost/internal/scheduler"
23+
"github.com/hookdeck/outpost/internal/telemetry"
2324
"go.uber.org/zap"
2425
)
2526

@@ -42,7 +43,7 @@ type APIService struct {
4243
consumerOptions *consumerOptions
4344
}
4445

45-
func NewService(ctx context.Context, wg *sync.WaitGroup, cfg *config.Config, logger *logging.Logger) (*APIService, error) {
46+
func NewService(ctx context.Context, wg *sync.WaitGroup, cfg *config.Config, logger *logging.Logger, telemetry telemetry.Telemetry) (*APIService, error) {
4647
wg.Add(1)
4748

4849
var cleanupFuncs []func(context.Context, *logging.LoggerWithCtx)
@@ -110,6 +111,7 @@ func NewService(ctx context.Context, wg *sync.WaitGroup, cfg *config.Config, log
110111
entityStore,
111112
logStore,
112113
eventHandler,
114+
telemetry,
113115
)
114116

115117
// deliverymqRetryScheduler

‎internal/services/api/destination_handlers.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,22 @@ import (
1111
"github.com/hookdeck/outpost/internal/destregistry"
1212
"github.com/hookdeck/outpost/internal/logging"
1313
"github.com/hookdeck/outpost/internal/models"
14+
"github.com/hookdeck/outpost/internal/telemetry"
1415
"github.com/hookdeck/outpost/internal/util/maputil"
1516
)
1617

1718
type DestinationHandlers struct {
1819
logger *logging.Logger
20+
telemetry telemetry.Telemetry
1921
entityStore models.EntityStore
2022
topics []string
2123
registry destregistry.Registry
2224
}
2325

24-
func NewDestinationHandlers(logger *logging.Logger, entityStore models.EntityStore, topics []string, registry destregistry.Registry) *DestinationHandlers {
26+
func NewDestinationHandlers(logger *logging.Logger, telemetry telemetry.Telemetry, entityStore models.EntityStore, topics []string, registry destregistry.Registry) *DestinationHandlers {
2527
return &DestinationHandlers{
2628
logger: logger,
29+
telemetry: telemetry,
2730
entityStore: entityStore,
2831
topics: topics,
2932
registry: registry,
@@ -97,6 +100,7 @@ func (h *DestinationHandlers) Create(c *gin.Context) {
97100
h.handleUpsertDestinationError(c, err)
98101
return
99102
}
103+
h.telemetry.DestinationCreated(c.Request.Context(), destination.Type)
100104

101105
display, err := h.registry.DisplayDestination(&destination)
102106
if err != nil {

‎internal/services/api/logger_middleware.go

+12
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77
"time"
88

9+
sentrygin "github.com/getsentry/sentry-go/gin"
910
"github.com/gin-gonic/gin"
1011
"github.com/hookdeck/outpost/internal/logging"
1112
"github.com/pkg/errors"
@@ -25,6 +26,10 @@ func LoggerMiddleware(logger *logging.Logger) gin.HandlerFunc {
2526

2627
if c.Writer.Status() >= 500 {
2728
logger.Error("request completed", fields...)
29+
30+
if hub := sentrygin.GetHubFromContext(c); hub != nil {
31+
hub.CaptureException(getErrorWithStackTrace(c.Errors.Last().Err))
32+
}
2833
} else {
2934
logger.Info("request completed", fields...)
3035
}
@@ -137,3 +142,10 @@ func getErrorFields(err error) []zap.Field {
137142

138143
return fields
139144
}
145+
146+
func getErrorWithStackTrace(err error) error {
147+
if errResp, ok := err.(ErrorResponse); ok {
148+
return errResp.Err
149+
}
150+
return err
151+
}

0 commit comments

Comments
 (0)