diff --git a/docker-compose.yml b/docker-compose.yml index 90fdc7161f..d44c3c358a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,7 +96,7 @@ services: - primary environment: - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=true - - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource,grafana-pyroscope-app - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-changeme} - GF_AUTH_ANONYMOUS_ENABLED=true @@ -104,6 +104,16 @@ services: profiles: - debug + pyroscope: + image: grafana/pyroscope:latest + ports: + - '4040:4040' + restart: unless-stopped + networks: + - primary + profiles: + - debug + graphqlmetrics: image: ghcr.io/wundergraph/cosmo/graphqlmetrics:${DC_GRAPHQLMETRICS_VERSION:-latest} build: diff --git a/router-tests/go.mod b/router-tests/go.mod index 07a86f3f43..c2253da200 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -27,7 +27,7 @@ require ( github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects v0.0.0-20250715110703-10f2e5f9c79e github.com/wundergraph/cosmo/router v0.0.0-20250912064154-106e871ee32e github.com/wundergraph/cosmo/router-plugin v0.0.0-20250808194725-de123ba1c65e - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227 go.opentelemetry.io/otel v1.36.0 go.opentelemetry.io/otel/sdk v1.36.0 go.opentelemetry.io/otel/sdk/metric v1.36.0 diff --git a/router-tests/go.sum b/router-tests/go.sum index 390d7e1971..7b2e7a7128 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -352,8 +352,8 @@ github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTB github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226 h1:3g6KNCG4ydgnpZnIlCK7pmtv0FSge6ILUS5LjrNZNiI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227 h1:uia2rhiJt/TIqZbeEvqoy68tSs5MZM4kG1Ht+wjHrF8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= diff --git a/router/bench-ws-superload.js b/router/bench-ws-superload.js new file mode 100644 index 0000000000..a6a9c01481 --- /dev/null +++ b/router/bench-ws-superload.js @@ -0,0 +1,86 @@ +import ws from 'k6/ws'; +import { check } from 'k6'; + +/* + This script was originally intended to simulate an extremely high load on websocket connections. It may also be useful as a base for new websocket scenarios. +*/ + +export const options = { + stages: [ + { duration: '0s', target: 10000 }, + { duration: '5m', target: 10000 }, + ], +}; + +export default function () { + const url = 'ws://localhost:3002/graphql'; + const params = { + headers: { + 'Sec-WebSocket-Protocol': 'graphql-transport-ws', + }, + }; + + const res = ws.connect(url, params, function (socket) { + socket.on('open', () => { + // Send connection_init message + socket.send( + JSON.stringify({ + type: 'connection_init', + }), + ); + }); + + socket.on('message', function (message) { + const data = JSON.parse(message); + + console.log(message); + + switch (data.type) { + case 'connection_ack': + // Connection acknowledged, start subscription + socket.send( + JSON.stringify({ + id: '1', + type: 'subscribe', + payload: { + query: 'subscription { countHob(max: 50000, intervalMilliseconds: 1) }', + }, + }), + ); + console.log('Subscription started'); + break; + case 'next': + console.log('Subscription next:', data.payload); + break; + case 'complete': + console.log('Subscription completed'); + break; + } + }); + + socket.on('close', function () { + console.log('WebSocket connection closed'); + }); + + socket.on('error', function (e) { + if (e.error() != 'websocket: close sent') { + console.log('WebSocket error:', e.error()); + } + }); + }); + + // Cancel subscription after 20 seconds + setTimeout(() => { + socket.send( + JSON.stringify({ + id: '1', + type: 'complete', + }), + ); + socket.close(); + }, 20000); + + check(res, { + 'WebSocket connection established': (r) => r && r.status === 101, + }); +} diff --git a/router/cmd/main.go b/router/cmd/main.go index 547b80bb4a..6f16ec1474 100644 --- a/router/cmd/main.go +++ b/router/cmd/main.go @@ -8,9 +8,11 @@ import ( "log" "os" "os/signal" + "runtime" "syscall" "time" + "github.com/grafana/pyroscope-go" "github.com/joho/godotenv" "github.com/wundergraph/cosmo/router/core" "github.com/wundergraph/cosmo/router/internal/timex" @@ -27,9 +29,11 @@ var ( overrideEnvFlag = flag.String("override-env", os.Getenv("OVERRIDE_ENV"), "Path to .env file to override environment variables") routerVersion = flag.Bool("version", false, "Prints the version and dependency information") pprofListenAddr = flag.String("pprof-addr", os.Getenv("PPROF_ADDR"), "Address to listen for pprof requests. e.g. :6060 for localhost:6060") - memProfilePath = flag.String("memprofile", "", "Path to write memory profile. Memory is a snapshot taken at the time the program exits") - cpuProfilePath = flag.String("cpuprofile", "", "Path to write cpu profile. CPU is measured from when the program starts until the program exits") - help = flag.Bool("help", false, "Prints the help message") + pyroscopeAddr = flag.String("pyroscope-addr", os.Getenv("PYROSCOPE_ADDR"), "Address to use for pyroscope continuous profiling. e.g. http://localhost:4040") + + memProfilePath = flag.String("memprofile", "", "Path to write memory profile. Memory is a snapshot taken at the time the program exits") + cpuProfilePath = flag.String("cpuprofile", "", "Path to write cpu profile. CPU is measured from when the program starts until the program exits") + help = flag.Bool("help", false, "Prints the help message") // Register the custom flag types configPathFlag = newMultipleString("config", os.Getenv("CONFIG_PATH"), "Path to the router config file e.g. config.yaml, in case the path is a comma separated file list e.g. \"config.yaml,override.yaml\", the configs will be merged") @@ -92,6 +96,14 @@ func Main() { zap.String("service_version", core.Version), ) + if *pprofListenAddr != "" && *pyroscopeAddr != "" { + baseLogger.Fatal("Cannot use pprof and pyroscope at the same time") + } + + if *pyroscopeAddr != "" && (*cpuProfilePath != "" || *memProfilePath != "") { + baseLogger.Fatal("Cannot use --cpuprofile or --memprofile while Pyroscope is enabled") + } + // Start pprof server if address is provided if *pprofListenAddr != "" { pprofSvr := profile.NewServer(*pprofListenAddr, baseLogger) @@ -103,6 +115,40 @@ func Main() { profiler := profile.Start(baseLogger, *cpuProfilePath, *memProfilePath) defer profiler.Finish() + if *pyroscopeAddr != "" { + runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(5) + + logger := baseLogger.With(zap.String("component", "pyroscope")) + logger.Info("starting pyroscope server") + + pyro, err := pyroscope.Start(pyroscope.Config{ + ApplicationName: "wundergraph.cosmo.router", + ServerAddress: *pyroscopeAddr, + Logger: logger.Sugar(), + Tags: map[string]string{"hostname": os.Getenv("HOSTNAME")}, + + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + }) + if err != nil { + logger.Error("failed to start pyroscope", zap.Error(err)) + } + if pyro != nil { + defer pyro.Stop() + } + } + rs, err := core.NewRouterSupervisor(&core.RouterSupervisorOpts{ BaseLogger: baseLogger, ConfigFactory: func() (*config.Config, error) { diff --git a/router/go.mod b/router/go.mod index ae51779a4f..3586e50e8d 100644 --- a/router/go.mod +++ b/router/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/twmb/franz-go v1.16.1 - github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226 + github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227 // Do not upgrade, it renames attributes we rely on go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 go.opentelemetry.io/contrib/propagators/b3 v1.23.0 @@ -68,6 +68,7 @@ require ( github.com/goccy/go-json v0.10.3 github.com/google/go-containerregistry v0.20.3 github.com/google/uuid v1.6.0 + github.com/grafana/pyroscope-go v1.2.7 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.3 github.com/iancoleman/strcase v0.3.0 @@ -114,6 +115,7 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/router/go.sum b/router/go.sum index cdc81151aa..8609b6b370 100644 --- a/router/go.sum +++ b/router/go.sum @@ -124,6 +124,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -317,8 +321,8 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226 h1:3g6KNCG4ydgnpZnIlCK7pmtv0FSge6ILUS5LjrNZNiI= -github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.226/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227 h1:uia2rhiJt/TIqZbeEvqoy68tSs5MZM4kG1Ht+wjHrF8= +github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.227/go.mod h1:g1IFIylu5Fd9pKjzq0mDvpaKhEB/vkwLAIbGdX2djXU= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=