goroutine 1915 [running]: runtime/pprof.writeGoroutineStacks({0x7fbac075bc68, 0xc000946390}) /usr/local/go/src/runtime/pprof/pprof.go:693 +0x70 runtime/pprof.writeGoroutine({0x7fbac075bc68, 0xc000946390}, 0x0) /usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b runtime/pprof.(*Profile).WriteTo(0x200fc60, {0x7fbac075bc68, 0xc000946390}, 0xc) /usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b net/http/pprof.handler.ServeHTTP({0xc00171e101, 0x3919160}, {0x26503d0, 0xc000946390}, 0xc00171e0f4) /usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a net/http/pprof.Index({0x26503d0, 0xc000946390}, 0xc000a48d00) /usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e net/http.HandlerFunc.ServeHTTP(0x2297860, {0x26503d0, 0xc000946390}, 0x266fb88) /usr/local/go/src/net/http/server.go:2047 +0x2f net/http.(*ServeMux).ServeHTTP(0xc000a48c00, {0x26503d0, 0xc000946390}, 0xc000a48d00) /usr/local/go/src/net/http/server.go:2425 +0x149 github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001e0b40, {0x26503d0, 0xc000946390}, 0xc000a48b00) /src/loki/vendor/github.com/gorilla/mux/mux.go:210 +0x1cf github.com/weaveworks/common/middleware.Instrument.Wrap.func1.2({0x26503d0, 0xc000946390}) /src/loki/vendor/github.com/weaveworks/common/middleware/instrument.go:70 +0x39 github.com/felixge/httpsnoop.CaptureMetricsFn({0x264c5c0, 0xc000d11ea0}, 0xc00034f600) /src/loki/vendor/github.com/felixge/httpsnoop/capture_metrics.go:76 +0x212 github.com/weaveworks/common/middleware.Instrument.Wrap.func1({0x264c5c0, 0xc000d11ea0}, 0xc000a48b00) /src/loki/vendor/github.com/weaveworks/common/middleware/instrument.go:69 +0x35b net/http.HandlerFunc.ServeHTTP(0xc000a48b00, {0x264c5c0, 0xc000d11ea0}, 0x0) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/weaveworks/common/middleware.Log.Wrap.func1({0x26507f0, 0xc000d11d60}, 0xc000a48b00) /src/loki/vendor/github.com/weaveworks/common/middleware/logging.go:55 +0x287 net/http.HandlerFunc.ServeHTTP(0x40d547, {0x26507f0, 0xc000d11d60}, 0xc00092d301) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5({0x264e780, 0xc0005de700}, 0xc000a48a00) /src/loki/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:154 +0x62d net/http.HandlerFunc.ServeHTTP(0x266fae0, {0x264e780, 0xc0005de700}, 0x25eb6c8) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/loki.(*Loki).initServer.func2.1({0x264e780, 0xc0005de700}, 0xc000a48900) /src/loki/pkg/loki/modules.go:127 +0x2c4 net/http.HandlerFunc.ServeHTTP(0xc00171e10b, {0x264e780, 0xc0005de700}, 0x46928e) /usr/local/go/src/net/http/server.go:2047 +0x2f net/http.serverHandler.ServeHTTP({0xc0009460f0}, {0x264e780, 0xc0005de700}, 0xc000a48900) /usr/local/go/src/net/http/server.go:2879 +0x43b net/http.(*conn).serve(0xc000a34960, {0x266fb88, 0xc000a0cf60}) /usr/local/go/src/net/http/server.go:1930 +0xb08 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:3034 +0x4e8 goroutine 1 [select, 104 minutes]: github.com/grafana/dskit/services.(*Manager).AwaitStopped(0xc0009ffc20, {0x266fb18, 0xc0001aa418}) /src/loki/vendor/github.com/grafana/dskit/services/manager.go:145 +0x73 github.com/grafana/loki/pkg/loki.(*Loki).Run(0xc000918000, {0xc0008dd000}) /src/loki/pkg/loki/loki.go:411 +0x765 main.main() /src/loki/cmd/loki/main.go:105 +0xdd5 goroutine 164 [select, 2 minutes]: github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc0008e60d0, 0xc0001ac050) /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:153 +0x8f github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc0008e60d0) /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:148 +0x6d created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:87 +0x165 goroutine 60 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc00057c080}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc0008e5a40) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 40 [chan receive, 104 minutes]: github.com/baidubce/bce-sdk-go/util/log.NewLogger.func1() /src/loki/vendor/github.com/baidubce/bce-sdk-go/util/log/logger.go:362 +0x65 created by github.com/baidubce/bce-sdk-go/util/log.NewLogger /src/loki/vendor/github.com/baidubce/bce-sdk-go/util/log/logger.go:359 +0x107 goroutine 72 [select, 2 minutes]: github.com/grafana/loki/pkg/ingester.(*Ingester).loop(0xc000a62580) /src/loki/pkg/ingester/ingester.go:519 +0x116 created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting /src/loki/pkg/ingester/ingester.go:465 +0xd85 goroutine 87 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000164880) /src/loki/vendor/go.opencensus.io/stats/view/worker.go:276 +0xb9 created by go.opencensus.io/stats/view.init.0 /src/loki/vendor/go.opencensus.io/stats/view/worker.go:34 +0x92 goroutine 100 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 141 [select, 104 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a1e3c0) /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:77 +0xa7 created by google.golang.org/grpc.newCCBalancerWrapper /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:67 +0x246 goroutine 166 [select, 2 minutes]: github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc00081c000, 0x0) /src/loki/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xc5 created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn /src/loki/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x210 goroutine 167 [select]: github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc0001a3a40) /src/loki/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0xe5 created by github.com/uber/jaeger-client-go.NewRemoteReporter /src/loki/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x247 goroutine 196 [select, 2 minutes]: github.com/grafana/loki/pkg/storage/chunk/cache.(*FifoCache).runPruneJob(0xc0001e1140, 0x0, 0xc000a0a060) /src/loki/pkg/storage/chunk/cache/fifo_cache.go:220 +0xdf created by github.com/grafana/loki/pkg/storage/chunk/cache.NewFifoCache /src/loki/pkg/storage/chunk/cache/fifo_cache.go:209 +0x1159 goroutine 209 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 210 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 211 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 212 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 213 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 214 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 215 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 216 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 217 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 218 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 219 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 220 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 221 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 222 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 223 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 224 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).worker(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:141 +0x207 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:90 +0x1c5 goroutine 225 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 226 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 227 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 228 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 229 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 230 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 231 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 232 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 233 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 234 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 235 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 236 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 237 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 238 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 239 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 240 [select, 104 minutes]: github.com/grafana/loki/pkg/storage/chunk/fetcher.(*Fetcher).asyncWriteBackCacheQueueProcessLoop(0xc0006a9b90) /src/loki/pkg/storage/chunk/fetcher/fetcher.go:115 +0xa8 created by github.com/grafana/loki/pkg/storage/chunk/fetcher.New /src/loki/pkg/storage/chunk/fetcher/fetcher.go:97 +0x258 goroutine 241 [select, 6 minutes]: github.com/grafana/loki/pkg/storage/chunk/client/local.(*BoltIndexClient).loop(0xc000937ea0) /src/loki/pkg/storage/chunk/client/local/boltdb_index_client.go:82 +0x105 created by github.com/grafana/loki/pkg/storage/chunk/client/local.NewBoltDBIndexClient /src/loki/pkg/storage/chunk/client/local/boltdb_index_client.go:71 +0x115 goroutine 242 [select]: github.com/grafana/dskit/kv/consul.(*mockKV).loop(0xc000a1ec30) /src/loki/vendor/github.com/grafana/dskit/kv/consul/mock.go:87 +0xc5 created by github.com/grafana/dskit/kv/consul.NewInMemoryClientWithConfig /src/loki/vendor/github.com/grafana/dskit/kv/consul/mock.go:58 +0x21b goroutine 257 [select, 6 minutes]: github.com/grafana/loki/pkg/storage/stores/shipper/compactor/generationnumber.(*GenNumberLoader).loop(0xc000a22700) /src/loki/pkg/storage/stores/shipper/compactor/generationnumber/gennumber_loader.go:45 +0xa5 created by github.com/grafana/loki/pkg/storage/stores/shipper/compactor/generationnumber.NewGenNumberLoader /src/loki/pkg/storage/stores/shipper/compactor/generationnumber/gennumber_loader.go:37 +0x109 goroutine 269 [select]: github.com/grafana/dskit/services.NewTimerService.func1({0x266fae0, 0xc000a23340}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:31 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc000a34b40) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 290 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 291 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 292 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 293 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 294 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*Manager).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/manager.go:244 +0x66 created by github.com/grafana/dskit/services.(*Manager).AddListener /src/loki/vendor/github.com/grafana/dskit/services/manager.go:243 +0x1c6 goroutine 297 [select, 104 minutes]: github.com/grafana/loki/pkg/loki.NewServerService.func1({0x266fae0, 0xc000a940c0}) /src/loki/pkg/loki/modules.go:1068 +0xf8 github.com/grafana/dskit/services.(*BasicService).main(0xc0008e50e0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 338 [select, 6 minutes]: github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc000598d80) /src/loki/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go:332 +0xaa created by github.com/prometheus/prometheus/tsdb/wal.NewSize /src/loki/vendor/github.com/prometheus/prometheus/tsdb/wal/wal.go:301 +0x2cf goroutine 347 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 355 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 356 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 357 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 379 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 380 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 381 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 382 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*Manager).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/manager.go:244 +0x66 created by github.com/grafana/dskit/services.(*Manager).AddListener /src/loki/vendor/github.com/grafana/dskit/services/manager.go:243 +0x1c6 goroutine 383 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 384 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 385 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 386 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 387 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 388 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 389 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 390 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 391 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 392 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 393 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 394 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:344 +0x66 created by github.com/grafana/dskit/services.(*BasicService).AddListener /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:343 +0x1c5 goroutine 395 [chan receive, 104 minutes]: github.com/grafana/dskit/services.(*Manager).AddListener.func1() /src/loki/vendor/github.com/grafana/dskit/services/manager.go:244 +0x66 created by github.com/grafana/dskit/services.(*Manager).AddListener /src/loki/vendor/github.com/grafana/dskit/services/manager.go:243 +0x1c6 goroutine 396 [select, 104 minutes]: github.com/weaveworks/common/signals.(*Handler).Loop(0xc00092a8d0) /src/loki/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x192 github.com/grafana/loki/pkg/loki.(*Loki).Run.func4() /src/loki/pkg/loki/loki.go:400 +0x2f created by github.com/grafana/loki/pkg/loki.(*Loki).Run /src/loki/pkg/loki/loki.go:399 +0x72f goroutine 397 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476b40}, 0x4, 0xc0000e5d40) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0000e5da0, {0x266fae0, 0xc000476b40}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000937770, {0x266fae0, 0xc000476b40}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc0008e59a0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 398 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc0000dfeb0, {0x266fae0, 0xc000476b80}, 0x4, 0xc000180120) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0x936232, {0x266fae0, 0xc000476b80}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a1ebe0, {0x266fae0, 0xc000476b80}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc0008e5d60) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 399 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476bc0}, 0x4, 0xc000180780) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0001807e0, {0x266fae0, 0xc000476bc0}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a1fb30, {0x266fae0, 0xc000476bc0}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a34460) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 400 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc000a9deb0, {0x266fae0, 0xc000476c00}, 0x4, 0xc0001809c0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0x936232, {0x266fae0, 0xc000476c00}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a40d70, {0x266fae0, 0xc000476c00}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a34780) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 401 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476c40}, 0x4, 0xc000181b60) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc000181bc0, {0x266fae0, 0xc000476c40}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a59a90, {0x266fae0, 0xc000476c40}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a35720) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 402 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476c80}, 0x4, 0xc0000e47e0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0000e4b40, {0x266fae0, 0xc000476c80}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc0000a1bd0, {0x266fae0, 0xc000476c80}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc0001b6000) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 403 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc0000dceb0, {0x266fae0, 0xc000476cc0}, 0x4, 0xc0000e5920) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0x936232, {0x266fae0, 0xc000476cc0}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc0009369b0, {0x266fae0, 0xc000476cc0}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc0008e5400) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 404 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476d00}, 0x4, 0xc000180b40) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc000180de0, {0x266fae0, 0xc000476d00}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a41a40, {0x266fae0, 0xc000476d00}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a34aa0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 405 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc0000dbeb0, {0x266fae0, 0xc000476d40}, 0x4, 0xc000181020) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0x936232, {0x266fae0, 0xc000476d40}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a58780, {0x266fae0, 0xc000476d40}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a34f00) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 406 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc000aa7eb0, {0x266fae0, 0xc000476d80}, 0x4, 0xc0002ba600) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0x936232, {0x266fae0, 0xc000476d80}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000951040, {0x266fae0, 0xc000476d80}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000a8f680) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 407 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476dc0}, 0x4, 0xc0002bacc0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0002bad20, {0x266fae0, 0xc000476dc0}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc000a90dc0, {0x266fae0, 0xc000476dc0}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc000420280) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 408 [select, 104 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x1, {0x266fae0, 0xc000476e00}, 0x4, 0xc0002bb800) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0x96 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0002bb860, {0x266fae0, 0xc000476e00}) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x2c github.com/grafana/dskit/modules.(*moduleService).run(0xc0000a05f0, {0x266fae0, 0xc000476e00}) /src/loki/vendor/github.com/grafana/dskit/modules/module_service.go:94 +0x34 github.com/grafana/dskit/services.(*BasicService).main(0xc0003d0d20) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 410 [syscall, 104 minutes]: os/signal.signal_recv() /usr/local/go/src/runtime/sigqueue.go:169 +0x98 os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:24 +0x19 created by os/signal.Notify.func1.1 /usr/local/go/src/os/signal/signal.go:151 +0x2c goroutine 298 [chan receive, 104 minutes]: github.com/weaveworks/common/server.(*Server).Run(0xc000135880) /src/loki/vendor/github.com/weaveworks/common/server/server.go:482 +0x1c7 github.com/grafana/loki/pkg/loki.NewServerService.func1.1() /src/loki/pkg/loki/modules.go:1065 +0x65 created by github.com/grafana/loki/pkg/loki.NewServerService.func1 /src/loki/pkg/loki/modules.go:1063 +0x93 goroutine 299 [chan receive, 104 minutes]: github.com/grafana/loki/pkg/loki.ignoreSignalHandler.Loop(0xc000a16fd0) /src/loki/pkg/loki/modules.go:1105 +0x1b github.com/weaveworks/common/server.(*Server).Run.func1() /src/loki/vendor/github.com/weaveworks/common/server/server.go:442 +0x35 created by github.com/weaveworks/common/server.(*Server).Run /src/loki/vendor/github.com/weaveworks/common/server/server.go:441 +0x8f goroutine 300 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07eff10, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0002c2480, 0xc00005a800, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc0002c2480) /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc0002c2480) /usr/local/go/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc000124738) /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc000124738) /usr/local/go/src/net/tcpsock.go:262 +0x3d github.com/weaveworks/common/middleware.(*countingListener).Accept(0xc00004c940) /src/loki/vendor/github.com/weaveworks/common/middleware/counting_listener.go:22 +0x2d net/http.(*Server).Serve(0xc00040c460, {0x264c5f0, 0xc00004c940}) /usr/local/go/src/net/http/server.go:3002 +0x394 github.com/weaveworks/common/server.(*Server).Run.func2() /src/loki/vendor/github.com/weaveworks/common/server/server.go:452 +0x71 created by github.com/weaveworks/common/server.(*Server).Run /src/loki/vendor/github.com/weaveworks/common/server/server.go:449 +0xe8 goroutine 301 [IO wait, 34 minutes]: internal/poll.runtime_pollWait(0x7fbac07efe20, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0002c2500, 0xc00005f800, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc0002c2500) /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc0002c2500) /usr/local/go/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc000124780) /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc000124780) /usr/local/go/src/net/tcpsock.go:262 +0x3d github.com/weaveworks/common/middleware.(*countingListener).Accept(0xc00004c960) /src/loki/vendor/github.com/weaveworks/common/middleware/counting_listener.go:22 +0x2d google.golang.org/grpc.(*Server).Serve(0xc0002ec700, {0x264c5f0, 0xc00004c960}) /src/loki/vendor/google.golang.org/grpc/server.go:779 +0x362 github.com/weaveworks/common/server.(*Server).Run.func3() /src/loki/vendor/github.com/weaveworks/common/server/server.go:471 +0x3f created by github.com/weaveworks/common/server.(*Server).Run /src/loki/vendor/github.com/weaveworks/common/server/server.go:470 +0x1b2 goroutine 302 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc000a94180}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc000a34500) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 303 [select, 104 minutes]: github.com/grafana/dskit/kv/memberlist.(*KVInitService).running(0xc0006a98c0, {0x266fae0, 0xc000a94200}) /src/loki/vendor/github.com/grafana/dskit/kv/memberlist/kv_init_service.go:70 +0x98 github.com/grafana/dskit/services.(*BasicService).main(0xc0008e54a0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 304 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000a221d0, 0x6a1b) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0ea2b849c86f9d4) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000a1ec30, {0xc00012a240, 0xf}, 0xc000ab0900) /src/loki/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0xbae github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc0006a9e60, {0x266fb88, 0xc000aae330}, {0xc00012a240, 0xf}, 0xc0006d6cf0) /src/loki/vendor/github.com/grafana/dskit/kv/consul/client.go:238 +0x324 github.com/grafana/dskit/kv.(*prefixedKVClient).WatchKey(0xc00004d520, {0x266fb88, 0xc000aae330}, {0x21d995f, 0xc00058dcb8}, 0x4cc677) /src/loki/vendor/github.com/grafana/dskit/kv/prefix.go:45 +0x73 github.com/grafana/dskit/kv.metrics.WatchKey.func1({0x266fb88, 0xc000aae330}) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:98 +0x42 github.com/weaveworks/common/instrument.CollectedRequest({0x266fae0, 0xc000a942c0}, {0x21dfce4, 0x8}, {0x264c590, 0xc0001a0980}, 0x21dd89c, 0xc00058de00) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:174 +0x274 github.com/grafana/dskit/kv.metrics.WatchKey({{0x2681338, 0xc00004d520}, 0xc0001a0980}, {0x266fae0, 0xc000a942c0}, {0x21d995f, 0x1f13dc0}, 0x1) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:97 +0xc5 github.com/grafana/dskit/ring.(*Ring).loop(0xc000a202c0, {0x266fae0, 0xc000a942c0}) /src/loki/vendor/github.com/grafana/dskit/ring/ring.go:274 +0xbe github.com/grafana/dskit/services.(*BasicService).main(0xc0008e5e00) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 417 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc000a94740}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc000a34be0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 418 [select, 2 minutes]: github.com/grafana/loki/pkg/usagestats.(*Reporter).running(0xc0009fd880, {0x266fae0, 0xc000a947c0}) /src/loki/pkg/usagestats/reporter.go:274 +0x35b github.com/grafana/dskit/services.(*BasicService).main(0xc000a34820) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 419 [select, 104 minutes]: github.com/grafana/loki/pkg/ingester.(*Ingester).running(0xc000a62580, {0x266fae0, 0xc000a94b40}) /src/loki/pkg/ingester/ingester.go:471 +0xc5 github.com/grafana/dskit/services.(*BasicService).main(0xc000a8f860) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 61 [select]: github.com/grafana/loki/pkg/scheduler.(*Scheduler).running(0xc000a5c700, {0x266fae0, 0xc00057c100}) /src/loki/pkg/scheduler/scheduler.go:653 +0x1b2 github.com/grafana/dskit/services.(*BasicService).main(0xc000a35220) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 62 [select]: github.com/grafana/dskit/services.NewTimerService.func1({0x266fae0, 0xc00057c140}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:31 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc000a34fa0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 63 [select, 4 minutes]: github.com/grafana/dskit/services.NewTimerService.func1({0x266fae0, 0xc00057c180}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:31 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc000a35040) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 64 [select]: github.com/grafana/dskit/ring.(*BasicLifecycler).running(0xc00040d880, {0x266fae0, 0xc00057c1c0}) /src/loki/vendor/github.com/grafana/dskit/ring/basic_lifecycler.go:194 +0x12b github.com/grafana/dskit/services.(*BasicService).main(0xc000a350e0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 433 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000a221d0, 0x6a1d) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0ea2b849c87352e) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000a1ec30, {0xc0007d80a8, 0x14}, 0xc000ab0b40) /src/loki/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0xbae github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000598900, {0x266fb88, 0xc000576720}, {0xc0007d80a8, 0x14}, 0xc0007b8020) /src/loki/vendor/github.com/grafana/dskit/kv/consul/client.go:238 +0x324 github.com/grafana/dskit/kv.(*prefixedKVClient).WatchKey(0xc00004dec0, {0x266fb88, 0xc000576720}, {0x21e24c0, 0xc000ce1cb8}, 0x4cc677) /src/loki/vendor/github.com/grafana/dskit/kv/prefix.go:45 +0x73 github.com/grafana/dskit/kv.metrics.WatchKey.func1({0x266fb88, 0xc000576720}) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:98 +0x42 github.com/weaveworks/common/instrument.CollectedRequest({0x266fae0, 0xc00057c200}, {0x21dfce4, 0x8}, {0x264c590, 0xc0001a0ac8}, 0x21dd89c, 0xc000ce1e00) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:174 +0x274 github.com/grafana/dskit/kv.metrics.WatchKey({{0x2681338, 0xc00004dec0}, 0xc0001a0ac8}, {0x266fae0, 0xc00057c200}, {0x21e24c0, 0x1f13dc0}, 0x1) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:97 +0xc5 github.com/grafana/dskit/ring.(*Ring).loop(0xc000a21b80, {0x266fae0, 0xc00057c200}) /src/loki/vendor/github.com/grafana/dskit/ring/ring.go:274 +0xbe github.com/grafana/dskit/services.(*BasicService).main(0xc000a35180) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 464 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a951d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x7fbac088dbc8) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937950) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x7) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 457 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95010, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x44ad12) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc0009376d0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x0) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 458 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95050, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x7fbac082b5e0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937720) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 459 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95090, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x7fbac082b5e0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc0009377c0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x2) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 460 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a950d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x7fbac082b5e0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937810) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x3) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 461 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95110, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x7fbac082b5e0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937860) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x4) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 462 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95150, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x44ad12) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc0009378b0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x5) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 465 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95210, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x9369f9) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc0009379a0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x8) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 456 [select, 6 minutes]: github.com/grafana/loki/pkg/ingester.(*Checkpointer).Run(0xc000cd7f78) /src/loki/pkg/ingester/checkpoint.go:613 +0x132 github.com/grafana/loki/pkg/ingester.(*walWrapper).run(0xc0000c7570) /src/loki/pkg/ingester/wal.go:166 +0x21c created by github.com/grafana/loki/pkg/ingester.(*walWrapper).Start /src/loki/pkg/ingester/wal.go:102 +0x6f goroutine 463 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95190, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x439c80) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937900) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x6) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 434 [select, 104 minutes]: github.com/grafana/loki/pkg/distributor.(*Distributor).running(0xc00039ee00, {0x266fae0, 0xc00057c680}) /src/loki/pkg/distributor/distributor.go:194 +0x98 github.com/grafana/dskit/services.(*BasicService).main(0xc0003d1540) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 435 [select]: github.com/grafana/dskit/ring.(*Lifecycler).loop(0xc000075000, {0x266fae0, 0xc00057c6c0}) /src/loki/vendor/github.com/grafana/dskit/ring/lifecycler.go:410 +0x26f github.com/grafana/dskit/services.(*BasicService).main(0xc0003d1360) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 436 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000a221d0, 0x6a1c) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0ea2b849c8703b8) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000a1ec30, {0xc0007d8120, 0x16}, 0xc000d42cc0) /src/loki/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0xbae github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000599320, {0x266fb88, 0xc000577110}, {0xc0007d8120, 0x16}, 0xc0007b89f0) /src/loki/vendor/github.com/grafana/dskit/kv/consul/client.go:238 +0x324 github.com/grafana/dskit/kv.(*prefixedKVClient).WatchKey(0xc000a7bc80, {0x266fb88, 0xc000577110}, {0x21e5a5a, 0xc000ce3cb8}, 0x4cc677) /src/loki/vendor/github.com/grafana/dskit/kv/prefix.go:45 +0x73 github.com/grafana/dskit/kv.metrics.WatchKey.func1({0x266fb88, 0xc000577110}) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:98 +0x42 github.com/weaveworks/common/instrument.CollectedRequest({0x266fae0, 0xc00057c700}, {0x21dfce4, 0x8}, {0x264c590, 0xc000126628}, 0x21dd89c, 0xc000ce3e00) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:174 +0x274 github.com/grafana/dskit/kv.metrics.WatchKey({{0x2681338, 0xc000a7bc80}, 0xc000126628}, {0x266fae0, 0xc00057c700}, {0x21e5a5a, 0x1f13dc0}, 0x1) /src/loki/vendor/github.com/grafana/dskit/kv/metrics.go:97 +0xc5 github.com/grafana/dskit/ring.(*Ring).loop(0xc0002c9600, {0x266fae0, 0xc00057c700}) /src/loki/vendor/github.com/grafana/dskit/ring/ring.go:274 +0xbe github.com/grafana/dskit/services.(*BasicService).main(0xc0003d1400) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 437 [select]: github.com/grafana/dskit/services.NewTimerService.func1({0x266fae0, 0xc00057c740}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:31 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc0003d14a0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 466 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95250, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc0009379f0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x9) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 467 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95290, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937a40) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xa) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 468 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a952d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937a90) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xb) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 469 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95310, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937ae0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xc) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 470 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95350, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937b30) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xd) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 471 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95390, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937b80) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xe) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 472 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a953d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937bd0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0xf) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 473 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95410, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937c20) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x10) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 474 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95450, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937c70) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x11) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 475 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95490, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937cc0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x12) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 476 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a954d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937d10) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x13) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 477 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95510, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937d60) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x14) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 478 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95550, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937db0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x15) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 479 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95590, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937e00) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x16) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 480 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a955d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000937f90) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x17) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 481 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95610, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a58000) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x18) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 482 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95650, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a58050) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x19) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 483 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95690, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a580a0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1a) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 484 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a956d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a580f0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1b) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 485 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95710, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a58140) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1c) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 486 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95750, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a58190) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1d) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 487 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a95790, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a581e0) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1e) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 488 [sync.Cond.Wait, 104 minutes]: sync.runtime_notifyListWait(0xc000a957d0, 0x0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/util.(*PriorityQueue).Dequeue(0xc000a58230) /src/loki/pkg/util/priority_queue.go:115 +0x8b github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc000a62580, 0x1f) /src/loki/pkg/ingester/flush.go:136 +0xb2 created by github.com/grafana/loki/pkg/ingester.(*Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:47 +0x45 goroutine 489 [select]: github.com/grafana/dskit/ring.(*Lifecycler).loop(0xc000a86c00, {0x266fae0, 0xc000a95800}) /src/loki/vendor/github.com/grafana/dskit/ring/lifecycler.go:410 +0x26f github.com/grafana/dskit/services.(*BasicService).main(0xc000a8f7c0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 411 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc000a94040}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc0003d0a00) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 412 [select]: github.com/grafana/dskit/services.NewTimerService.func1({0x266fae0, 0xc000a94140}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:31 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc0003d08c0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 438 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc00092e080}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc000a8f220) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 413 [select]: github.com/grafana/loki/pkg/util.(*ringWatcher).watchLoop(0xc000a916d0, {0x266fae0, 0xc000a941c0}) /src/loki/pkg/util/ring_watcher.go:44 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc0003d0960) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 439 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc00092e100}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc000a35ae0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 440 [chan receive, 104 minutes]: github.com/grafana/dskit/services.NewIdleService.func1({0x266fae0, 0xc000476000}) /src/loki/vendor/github.com/grafana/dskit/services/services.go:13 +0x2e github.com/grafana/dskit/services.(*BasicService).main(0xc000a35a40) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 441 [select]: github.com/grafana/loki/pkg/util.(*ringWatcher).watchLoop(0xc000a59d10, {0x266fae0, 0xc000476040}) /src/loki/pkg/util/ring_watcher.go:44 +0xfd github.com/grafana/dskit/services.(*BasicService).main(0xc000a359a0) /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x1e8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 /src/loki/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0xbe goroutine 143 [select, 8 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000288af0, {0xc000282610, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:190 +0xaa google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000288af0, {0xc000282610, 0xc0001cc408, 0xc00032b940}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:170 +0x147 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0003cf1a0, {0xc000282610, 0xc00032b9b8, 0x9bb307}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc0003cf1a0}, {0xc000282610, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000318240, {0xc000282610, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000282600, 0x6400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x394c088, 0xc000318240, {0x0, 0x0}, 0x6400000, 0x0, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc000054301, {0x7fbac0639fa0, 0x394c088}, 0xc, {0x0, 0x0}, {0x214ba20, 0xc00037f900}, 0xc0005c4050, 0x0, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*csAttempt).recvMsg(0xc00047c6e0, {0x214ba20, 0xc00037f900}, 0x8) /src/loki/vendor/google.golang.org/grpc/stream.go:975 +0x2b0 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc000a48000, 0xc00032bda8, 0xc00032bd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc000a48000, {0x214ba20, 0xc00037f900}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000551600) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000551600}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c140}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 144 [select, 6 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000288870, {0xc0002824a8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:190 +0xaa google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000288870, {0xc0002824a8, 0xc0001cc3d8, 0xc00032d940}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:170 +0x147 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0003cee70, {0xc0002824a8, 0xc00032d9b8, 0x9bb307}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc0003cee70}, {0xc0002824a8, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000a33c20, {0xc0002824a8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000282498, 0x6400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x394c088, 0xc000a33c20, {0x0, 0x0}, 0x6400000, 0x0, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc000054201, {0x7fbac0639fa0, 0x394c088}, 0xc, {0x0, 0x0}, {0x214ba20, 0xc0003e67c0}, 0xc0005c4040, 0x0, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*csAttempt).recvMsg(0xc00047c790, {0x214ba20, 0xc0003e67c0}, 0x8) /src/loki/vendor/google.golang.org/grpc/stream.go:975 +0x2b0 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc000a48100, 0xc00032dda8, 0xc00032dd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc000a48100, {0x214ba20, 0xc0003e67c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000550ea0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000550ea0}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c200}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 497 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000318900) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000bb550, {0x214ba20, 0xc0005563c0}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0005a0500, 0xc0003e1da8, 0xc0003e1d78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005a0500, {0x214ba20, 0xc0005563c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000551800) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000551800}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c240}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 498 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a33e60) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc000824000, {0x214ba20, 0xc00033fb40}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00026e000, 0xc0005e7da8, 0xc0005e7d78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00026e000, {0x214ba20, 0xc00033fb40}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000551040) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000551040}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c280}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 499 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a32120) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000bb340, {0x214ba20, 0xc000057900}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0005a0200, 0xc00058fda8, 0xc00058fd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005a0200, {0x214ba20, 0xc000057900}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc00018bbe0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc00018bbe0}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c2c0}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 500 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0003186c0) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000bb3f0, {0x214ba20, 0xc00035c9c0}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0005a0300, 0xc000355da8, 0xc000355d78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005a0300, {0x214ba20, 0xc00035c9c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc0005516d0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc0005516d0}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c340}, 0xc000373230, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 501 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0009500f0, {0xc000a520b8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:190 +0xaa google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0009500f0, {0xc000a520b8, 0xc000900000, 0xc00032f940}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:170 +0x147 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00072aae0, {0xc000a520b8, 0xc00032f9b8, 0x9bb307}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc00072aae0}, {0xc000a520b8, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0006d87e0, {0xc000a520b8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000a520a8, 0x6400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x394c088, 0xc0006d87e0, {0x0, 0x0}, 0x6400000, 0x0, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc000141001, {0x7fbac0639fa0, 0x394c088}, 0xa, {0x0, 0x0}, {0x214ba20, 0xc00037f4c0}, 0xc000832000, 0x0, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000bb290, {0x214ba20, 0xc00037f4c0}, 0x8) /src/loki/vendor/google.golang.org/grpc/stream.go:975 +0x2b0 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0005a0100, 0xc00032fda8, 0xc00032fd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005a0100, {0x214ba20, 0xc00037f4c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000128f60) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000128f60}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c380}, 0x21fa562, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 502 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a32d80) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000bb4a0, {0x214ba20, 0xc00033f9c0}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0005a0400, 0xc000329da8, 0xc000329d78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005a0400, {0x214ba20, 0xc00033f9c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc000550bd0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc000550bd0}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c400}, 0xc000053f38, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 503 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000288550, {0xc000282340, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:190 +0xaa google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000288550, {0xc000282340, 0xc0001cc390, 0xc0003dd940}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:170 +0x147 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0003ce4b0, {0xc000282340, 0xc0003dd9b8, 0x9bb307}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc0003ce4b0}, {0xc000282340, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000a32a20, {0xc000282340, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000282330, 0x6400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x394c088, 0xc000a32a20, {0x0, 0x0}, 0x6400000, 0x0, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc000141e01, {0x7fbac0639fa0, 0x394c088}, 0xc, {0x0, 0x0}, {0x214ba20, 0xc0003e6500}, 0xc0005c4030, 0x0, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*csAttempt).recvMsg(0xc00047c8f0, {0x214ba20, 0xc0003e6500}, 0x8) /src/loki/vendor/google.golang.org/grpc/stream.go:975 +0x2b0 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc000a48200, 0xc0003ddda8, 0xc0003ddd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc000a48200, {0x214ba20, 0xc0003e6500}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc00018be90) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc00018be90}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c440}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 504 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a327e0) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:323 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:338 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00017f340, {0x214ba20, 0xc00033f840}, 0x90) /src/loki/vendor/google.golang.org/grpc/stream.go:961 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000) /src/loki/vendor/google.golang.org/grpc/stream.go:826 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc000a60200, 0xc000cdfda8, 0xc000cdfd78) /src/loki/vendor/google.golang.org/grpc/stream.go:680 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc000a60200, {0x214ba20, 0xc00033f840}) /src/loki/vendor/google.golang.org/grpc/stream.go:825 +0x11f github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForQuerierQuerierLoopClient).Recv(0xc00018be20) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:811 +0x4c github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).querierLoop(0xc00040a1e0, {0x269efb0, 0xc00018be20}, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:112 +0x86 github.com/grafana/loki/pkg/querier/worker.(*schedulerProcessor).processQueriesOnSingleStream(0xc00040a1e0, {0x266fae0, 0xc000a7c480}, 0x0, {0xc000578140, 0xe}) /src/loki/pkg/querier/worker/scheduler_processor.go:90 +0x372 github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency.func1() /src/loki/pkg/querier/worker/processor_manager.go:78 +0xaf created by github.com/grafana/loki/pkg/querier/worker.(*processorManager).concurrency /src/loki/pkg/querier/worker/processor_manager.go:72 +0xce goroutine 74 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00000a3c0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1573 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:359 +0x18e5 goroutine 80 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 75 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7fbac07efd30, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0001b4100, 0xc0005c6000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0001b4100, {0xc0005c6000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0001b4100, {0xc0005c6000, 0x60100000000, 0x8}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0005c4000, {0xc0005c6000, 0x439c80, 0xc0002d0680}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc00075e180, {0xc0005de040, 0x9, 0xc0005b5d18}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc00075e180}, {0xc0005de040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0005de040, 0x9, 0x18}, {0x2628900, 0xc00075e180}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005de000) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000a3c0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1499 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:364 +0x192f goroutine 76 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000288140, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00075e2a0) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:414 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:412 +0x1f85 goroutine 77 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a40960, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00075e420) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:326 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:323 +0x180f goroutine 78 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000a291e0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1125 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:335 +0x1857 goroutine 79 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7fbac07efc40, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a04480, 0xc0008ce000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a04480, {0xc0008ce000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a04480, {0xc0008ce000, 0x9a3a6c, 0x800010601}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc00000e0a8, {0xc0008ce000, 0x98a288, 0xc000900b40}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc000a5b080, {0xc00040c660, 0x9, 0x5ae630647ac}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc000a5b080}, {0xc00040c660, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00040c660, 0x9, 0xc000a409b0}, {0x2628900, 0xc000a5b080}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00040c620) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000a291e0, 0xc00052e1e0, 0x1d3faa0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:605 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc0002ec700, {0x26a25b8, 0xc000a291e0}) /src/loki/vendor/google.golang.org/grpc/server.go:905 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() /src/loki/vendor/google.golang.org/grpc/server.go:847 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn /src/loki/vendor/google.golang.org/grpc/server.go:846 +0x185 goroutine 513 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 514 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 515 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 516 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 517 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 518 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 519 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 520 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 505 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc0003cf5c0}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc0003cf5c0}, {0xc0002d34d0}, {0xc0002a8180, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc0006d6d70}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ac00}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ac00}, 0x2096cc0, 0xc0003db758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ac00}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7abe0}, 0xc0003db7b8, 0xc000a7aae0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7abe0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7abe0}, 0xc0002829a8, 0xc000a7ab00) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7abe0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a0c0}, 0xc0002829a8, 0xc000a7ab20) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a0c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a0c0}, 0xc0002829a8, 0xc000a7ab40) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a0c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a0c0}, 0x1d3fa20, 0xc0006d6d40) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab0e0, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab0e0, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 506 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc0003cfdd0}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc0003cfdd0}, {0x2}, {0xc0002a8340, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc0006d6f00}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b0a0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b0a0}, 0x2096cc0, 0xc0005e1758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b0a0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b040}, 0xc0005e17b8, 0xc000a7aee0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b040}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b040}, 0xc000282a38, 0xc000a7af40) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b040}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a3c0}, 0xc000282a38, 0xc000a7af60) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a3c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a3c0}, 0xc000282a38, 0xc000a7af80) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a3c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a3c0}, 0x1d3fa20, 0xc0006d6ed0) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab320, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab320, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 507 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc0008cc1e0}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc0008cc1e0}, {0x2}, {0xc0002a83f0, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc0006d7050}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b2a0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b2a0}, 0x2096cc0, 0xc000ac5758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b2a0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b280}, 0xc000ac57b8, 0xc000a7b100) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b280}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b280}, 0xc000282a68, 0xc000a7b160) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b280}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a540}, 0xc000282a68, 0xc000a7b180) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a540}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a540}, 0xc000282a68, 0xc000a7b1e0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a540}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a540}, 0x1d3fa20, 0xc0006d7020) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab440, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006ab440, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 508 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc0008cc690}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc0008cc690}, {0xc0002d2d80}, {0xc0002a8440, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc0006d7160}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b520}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b520}, 0x2096cc0, 0xc000ac7758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7b520}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b500}, 0xc000ac77b8, 0xc000a7b340) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b500}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b500}, 0xc000282a98, 0xc000a7b360) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7b500}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a6c0}, 0xc000282a98, 0xc000a7b3c0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a6c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a6c0}, 0xc000282a98, 0xc000a7b3e0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a6c0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a6c0}, 0x1d3fa20, 0xc0006d7110) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006abc20, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006abc20, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 509 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc0003cfa70}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc0003cfa70}, {0x2}, {0xc0002a8214, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc0006d6e70}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ae80}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ae80}, 0x2096cc0, 0xc000ab9758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000a7ae80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7ae60}, 0xc000ab97b8, 0xc000a7acc0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7ae60}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7ae60}, 0xc000282a08, 0xc000a7ace0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000a7ae60}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a240}, 0xc000282a08, 0xc000a7ad00) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a240}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a240}, 0xc000282a08, 0xc000a7ad20) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a240}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a240}, 0x1d3fa20, 0xc0006d6e40) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006abd40, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0006abd40, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 510 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc00059a900}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc00059a900}, {0xc0007bb680}, {0xc0003ca300, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc00007fa00}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116fa0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116fa0}, 0x2096cc0, 0xc0005e5758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116fa0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116f80}, 0xc0005e57b8, 0xc000116e80) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116f80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116f80}, 0xc00000c7e0, 0xc000116ea0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116f80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e900}, 0xc00000c7e0, 0xc000116ec0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e900}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x20}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e900}, 0xc00000c7e0, 0xc000116ee0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e900}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e900}, 0x1d3fa20, 0xc00007f9d0) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc00051f8c0, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc00051f8c0, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 511 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc00059ad20}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc00059ad20}, {0x2}, {0xc0003ca350, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc00007fa90}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117120}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117120}, 0x2096cc0, 0xc000ab5758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117120}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117100}, 0xc000ab57b8, 0xc000117000) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117100}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117100}, 0xc00000c810, 0xc000117020) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117100}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ea80}, 0xc00000c810, 0xc000117040) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ea80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x20}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ea80}, 0xc00000c810, 0xc000117060) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ea80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ea80}, 0x1d3fa20, 0xc00007fa60) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2000, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2000, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 512 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc00059b080}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc00059b080}, {0xc0007bb830}, {0xc0003ca3a0, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc00007fb20}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001172a0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001172a0}, 0x2096cc0, 0xc000ab7758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001172a0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117280}, 0xc000ab77b8, 0xc000117180) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117280}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117280}, 0xc00000c840, 0xc0001171a0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117280}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ec00}, 0xc00000c840, 0xc0001171c0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ec00}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x20}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ec00}, 0xc00000c840, 0xc0001171e0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ec00}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ec00}, 0x1d3fa20, 0xc00007faf0) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2240, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2240, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 529 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc00059b440}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc00059b440}, {0x2}, {0xc0003ca3e0, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc00007fbb0}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117420}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117420}, 0x2096cc0, 0xc000ac9758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000117420}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117400}, 0xc000ac97b8, 0xc000117300) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117400}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117400}, 0xc00000c870, 0xc000117320) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000117400}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ed80}, 0xc00000c870, 0xc000117340) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ed80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x20}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ed80}, 0xc00000c870, 0xc000117360) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ed80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038ed80}, 0x1d3fa20, 0xc00007fb80) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d25a0, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d25a0, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 530 [select, 2 minutes]: github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait({0xc000a237c0, 0x0}, {0x266fb88, 0xc00059a750}) /src/loki/pkg/scheduler/queue/queue.go:258 +0x1c5 github.com/grafana/loki/pkg/scheduler/queue.(*RequestQueue).GetNextRequestForQuerier(0xc000a58910, {0x266fb88, 0xc00059a750}, {0x2}, {0xc0003ca2c0, 0xc}) /src/loki/pkg/scheduler/queue/queue.go:124 +0x1af github.com/grafana/loki/pkg/scheduler.(*Scheduler).QuerierLoop(0xc000a5c700, {0x269f008, 0xc00007f970}) /src/loki/pkg/scheduler/scheduler.go:453 +0x2f7 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForQuerier_QuerierLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116de0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:855 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116de0}, 0x2096cc0, 0xc0005e3758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc000116de0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116dc0}, 0xc0005e37b8, 0xc000116cc0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116dc0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116dc0}, 0xc00000c7b0, 0xc000116ce0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc000116dc0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e780}, 0xc00000c7b0, 0xc000116d00) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e780}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x20}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e780}, 0xc00000c7b0, 0xc000116d20) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e780}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00038e780}, 0x1d3fa20, 0xc00007f940) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2900, 0xc000a4f260, 0x38e7b60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000a291e0}, 0xc0000d2900, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 1951 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x36) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0002bb920) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1996 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x3b) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1949 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x35) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0000e4a20) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1987 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x38) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0008de660) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1985 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x37) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0008de720) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1991 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x3a) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0008dea80) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1989 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x39) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0008de8a0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 541 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 547 [select, 104 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a905f0) /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:77 +0xa7 created by google.golang.org/grpc.newCCBalancerWrapper /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:67 +0x246 goroutine 1972 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x32) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc000830f60) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1945 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x33) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc000831020) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 1947 [sync.Cond.Wait, 2 minutes]: sync.runtime_notifyListWait(0xc000a237d0, 0x34) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0008310e0) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait.func1() /src/loki/pkg/scheduler/queue/queue.go:240 +0x45 created by github.com/grafana/loki/pkg/scheduler/queue.contextCond.Wait /src/loki/pkg/scheduler/queue/queue.go:236 +0xd3 goroutine 561 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0008ee000) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1573 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:359 +0x18e5 goroutine 593 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 562 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07efb50, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a04580, 0xc000cb4000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a04580, {0xc000cb4000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a04580, {0xc000cb4000, 0x1060100000000, 0x8}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0008c8000, {0xc000cb4000, 0x4059e0, 0xc000a5a1e0}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc000572b40, {0xc0008e0040, 0x9, 0x18}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc000572b40}, {0xc0008e0040, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0008e0040, 0x9, 0xc0001424c8}, {0x2628900, 0xc000572b40}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008e0000) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0008ee000) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1499 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:364 +0x192f goroutine 563 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0000a01e0, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000572ba0) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:414 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:412 +0x1f85 goroutine 494 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a40af0, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000a5b2c0) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:326 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:323 +0x180f goroutine 495 [select, 104 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000596000) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1125 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:335 +0x1857 goroutine 496 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07efa60, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a04600, 0xc000cac000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a04600, {0xc000cac000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a04600, {0xc000cac000, 0x46928e, 0x439c47}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc00000e0b0, {0xc000cac000, 0x98a288, 0xc0001424b0}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc000a5b260, {0xc00040c740, 0x9, 0xc0005b7d30}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc000a5b260}, {0xc00040c740, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00040c740, 0x9, 0x5aff4cba870}, {0x2628900, 0xc000a5b260}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00040c700) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000596000, 0x0, 0x0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:605 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc0002ec700, {0x26a25b8, 0xc000596000}) /src/loki/vendor/google.golang.org/grpc/server.go:905 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() /src/loki/vendor/google.golang.org/grpc/server.go:847 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn /src/loki/vendor/google.golang.org/grpc/server.go:846 +0x185 goroutine 414 [select, 4 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).schedulerLoop(0xc0001b4080, {0x269ef00, 0xc000474050}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:269 +0x218 github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).runOne(0xc0001b4080, {0x266fae0, 0xc0008ec080}, {0x262afc0, 0xc0005c4008}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:236 +0x2ef github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start.func1() /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:208 +0x6e created by github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:206 +0x65 goroutine 415 [select, 4 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).schedulerLoop(0xc0001b4080, {0x269ef00, 0xc000952030}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:269 +0x218 github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).runOne(0xc0001b4080, {0x266fae0, 0xc0008ec080}, {0x262afc0, 0xc0005c4008}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:236 +0x2ef github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start.func1() /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:208 +0x6e created by github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:206 +0x65 goroutine 416 [select, 6 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).schedulerLoop(0xc0001b4080, {0x269ef00, 0xc00030a2d0}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:269 +0x218 github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).runOne(0xc0001b4080, {0x266fae0, 0xc0008ec080}, {0x262afc0, 0xc0005c4008}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:236 +0x2ef github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start.func1() /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:208 +0x6e created by github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:206 +0x65 goroutine 577 [select, 2 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).schedulerLoop(0xc0001b4080, {0x269ef00, 0xc0007b8130}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:269 +0x218 github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).runOne(0xc0001b4080, {0x266fae0, 0xc0008ec080}, {0x262afc0, 0xc0005c4008}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:236 +0x2ef github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start.func1() /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:208 +0x6e created by github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:206 +0x65 goroutine 578 [select, 4 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).schedulerLoop(0xc0001b4080, {0x269ef00, 0xc00037a080}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:269 +0x218 github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).runOne(0xc0001b4080, {0x266fae0, 0xc0008ec080}, {0x262afc0, 0xc0005c4008}) /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:236 +0x2ef github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start.func1() /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:208 +0x6e created by github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*frontendSchedulerWorker).start /src/loki/pkg/lokifrontend/frontend/v2/frontend_scheduler_worker.go:206 +0x65 goroutine 579 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 549 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 442 [select, 104 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() /src/loki/vendor/google.golang.org/grpc/stream.go:333 +0x98 created by google.golang.org/grpc.newClientStreamWithParams /src/loki/vendor/google.golang.org/grpc/stream.go:332 +0xb45 goroutine 594 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a40460, {0xc0001245f8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:178 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a40460, {0xc0001245f8, 0xc0000520c0, 0xc0003f70f8}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:172 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0005777d0, {0xc0001245f8, 0xc0003f7170, 0x9ca1a7}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc0005777d0}, {0xc0001245f8, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008ea900, {0xc0001245f8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0001245e8, 0x400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc0003f7328, 0xc0008ea900, {0x0, 0x0}, 0x400000, 0xc0003f7420, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fbac0639fa0, 0x394c088}, 0x8, {0x0, 0x0}, {0x21554c0, 0xc0008dc400}, 0xc0008dc400, 0xc0003f7420, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00035a300, {0x21554c0, 0xc0008dc400}) /src/loki/vendor/google.golang.org/grpc/stream.go:1563 +0x185 github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForFrontendFrontendLoopServer).Recv(0xc0007b93f0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:1013 +0x4c github.com/grafana/loki/pkg/scheduler.(*Scheduler).FrontendLoop(0xc000a5c700, {0x269ef58, 0xc0007b93f0}) /src/loki/pkg/scheduler/scheduler.go:282 +0x214 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForFrontend_FrontendLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ee0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:994 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ee0}, 0x2096cc0, 0xc0003f7758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ee0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7d40}, 0xc0003f77b8, 0xc0001a7be0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7d40}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7d40}, 0xc000124600, 0xc0001a7c20) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7d40}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a300}, 0xc000124600, 0xc0001a7c40) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a300}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a300}, 0xc000124600, 0xc0001a7c60) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a300}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a300}, 0x1d3fa20, 0xc0007b9380) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008ea900, 0xc000a4f1d0, 0x38e7b80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008ea900, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 595 [select, 6 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a40500, {0xc000a520a0, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:178 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a40500, {0xc000a520a0, 0xc0000520d8, 0xc0008350f8}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:172 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0005779b0, {0xc000a520a0, 0xc000835170, 0x9ca1a7}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc0005779b0}, {0xc000a520a0, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008eaa20, {0xc000a520a0, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000a52090, 0x400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc000835328, 0xc0008eaa20, {0x0, 0x0}, 0x400000, 0xc000835420, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fbac0639fa0, 0x394c088}, 0x8, {0x0, 0x0}, {0x21554c0, 0xc000370800}, 0xc000370800, 0xc000835420, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc000574000, {0x21554c0, 0xc000370800}) /src/loki/vendor/google.golang.org/grpc/stream.go:1563 +0x185 github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForFrontendFrontendLoopServer).Recv(0xc0008ca180) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:1013 +0x4c github.com/grafana/loki/pkg/scheduler.(*Scheduler).FrontendLoop(0xc000a5c700, {0x269ef58, 0xc0008ca180}) /src/loki/pkg/scheduler/scheduler.go:282 +0x214 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForFrontend_FrontendLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0002a6180}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:994 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0002a6180}, 0x2096cc0, 0xc000835758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0002a6180}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0002a6160}, 0xc0008357b8, 0xc0002a6020) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0002a6160}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0002a6160}, 0xc000a520c0, 0xc0002a6040) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0002a6160}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000574000}, 0xc000a520c0, 0xc0002a6060) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000574000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x44}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000574000}, 0xc000a520c0, 0xc0002a6080) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000574000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000574000}, 0x1d3fa20, 0xc0008ca150) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eaa20, 0xc000a4f1d0, 0x38e7b80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eaa20, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 596 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a405a0, {0xc0008d6088, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:178 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a405a0, {0xc0008d6088, 0xc0000520f0, 0xc0008390f8}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:172 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000577b90, {0xc0008d6088, 0xc000839170, 0x9ca1a7}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc000577b90}, {0xc0008d6088, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008eab40, {0xc0008d6088, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0008d6078, 0x400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc000839328, 0xc0008eab40, {0x0, 0x0}, 0x400000, 0xc000839420, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fbac0639fa0, 0x394c088}, 0x8, {0x0, 0x0}, {0x21554c0, 0xc000476f40}, 0xc000476f40, 0xc000839420, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc000ab0000, {0x21554c0, 0xc000476f40}) /src/loki/vendor/google.golang.org/grpc/stream.go:1563 +0x185 github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForFrontendFrontendLoopServer).Recv(0xc0004fa3a0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:1013 +0x4c github.com/grafana/loki/pkg/scheduler.(*Scheduler).FrontendLoop(0xc000a5c700, {0x269ef58, 0xc0004fa3a0}) /src/loki/pkg/scheduler/scheduler.go:282 +0x214 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForFrontend_FrontendLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc00082e1a0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:994 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc00082e1a0}, 0x2096cc0, 0xc000839758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc00082e1a0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc00082e180}, 0xc0008397b8, 0xc00082e020) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc00082e180}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc00082e180}, 0xc0008d6090, 0xc00082e060) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc00082e180}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000ab0000}, 0xc0008d6090, 0xc00082e0a0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000ab0000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x24}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000ab0000}, 0xc0008d6090, 0xc00082e0e0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000ab0000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc000ab0000}, 0x1d3fa20, 0xc0004fa080) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eab40, 0xc000a4f1d0, 0x38e7b80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eab40, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 597 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a40780, {0xc0001245c8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:178 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a40780, {0xc0001245c8, 0xc000052108, 0xc0003fd0f8}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:172 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000577d70, {0xc0001245c8, 0xc0003fd170, 0x9ca1a7}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc000577d70}, {0xc0001245c8, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008eac60, {0xc0001245c8, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0001245b8, 0x400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc0003fd328, 0xc0008eac60, {0x0, 0x0}, 0x400000, 0xc0003fd420, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fbac0639fa0, 0x394c088}, 0x8, {0x0, 0x0}, {0x21554c0, 0xc000371340}, 0xc000371340, 0xc0003fd420, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00035a180, {0x21554c0, 0xc000371340}) /src/loki/vendor/google.golang.org/grpc/stream.go:1563 +0x185 github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForFrontendFrontendLoopServer).Recv(0xc0007b9370) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:1013 +0x4c github.com/grafana/loki/pkg/scheduler.(*Scheduler).FrontendLoop(0xc000a5c700, {0x269ef58, 0xc0007b9370}) /src/loki/pkg/scheduler/scheduler.go:282 +0x214 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForFrontend_FrontendLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ba0}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:994 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ba0}, 0x2096cc0, 0xc0003fd758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7ba0}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7b80}, 0xc0003fd7b8, 0xc0001a7a40) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7b80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7b80}, 0xc0001245d0, 0xc0001a7a60) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7b80}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a180}, 0xc0001245d0, 0xc0001a7aa0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a180}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a180}, 0xc0001245d0, 0xc0001a7ac0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a180}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a180}, 0x1d3fa20, 0xc0007b92f0) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eac60, 0xc000a4f1d0, 0x38e7b80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008eac60, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 598 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a40820, {0xc000124598, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:178 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a40820, {0xc000124598, 0xc000052120, 0xc0003fb0f8}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:172 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000577f50, {0xc000124598, 0xc0003fb170, 0x9ca1a7}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:484 +0x32 io.ReadAtLeast({0x262f4c0, 0xc000577f50}, {0xc000124598, 0x5, 0x5}, 0x5) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008ead80, {0xc000124598, 0x5, 0x5}) /src/loki/vendor/google.golang.org/grpc/internal/transport/transport.go:468 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc000124588, 0x400000) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc0003fb328, 0xc0008ead80, {0x0, 0x0}, 0x400000, 0xc0003fb420, {0x0, 0x0}) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fbac0639fa0, 0x394c088}, 0x8, {0x0, 0x0}, {0x21554c0, 0xc0003662c0}, 0xc0003662c0, 0xc0003fb420, ...) /src/loki/vendor/google.golang.org/grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00035a000, {0x21554c0, 0xc0003662c0}) /src/loki/vendor/google.golang.org/grpc/stream.go:1563 +0x185 github.com/grafana/loki/pkg/scheduler/schedulerpb.(*schedulerForFrontendFrontendLoopServer).Recv(0xc0007b92e0) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:1013 +0x4c github.com/grafana/loki/pkg/scheduler.(*Scheduler).FrontendLoop(0xc000a5c700, {0x269ef58, 0xc0007b92e0}) /src/loki/pkg/scheduler/scheduler.go:282 +0x214 github.com/grafana/loki/pkg/scheduler/schedulerpb._SchedulerForFrontend_FrontendLoop_Handler({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7680}) /src/loki/pkg/scheduler/schedulerpb/scheduler.pb.go:994 +0x9f github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7680}, 0x2096cc0, 0xc0003fb758) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:51 +0xb4 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26836a8, 0xc0001a7680}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grafana/loki/pkg/util/fakeauth.glob..func3({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7660}, 0xc0003fb7b8, 0xc0001a73a0) /src/loki/pkg/util/fakeauth/fake_auth.go:65 +0xfa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7660}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7660}, 0xc0001245a0, 0xc0001a73c0) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:45 +0xaa github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x2682340, 0xc0001a7660}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a000}, 0xc0001245a0, 0xc0001a73e0) /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x382 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor({{0x26ad620, 0xc0007b8ba0}, 0x0, 0x0}, {0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a000}, 0xc0001245a0, 0xc0001a7400) /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:54 +0xc2 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a000}) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x3a github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x216f6a0, 0xc000a5c700}, {0x26826e8, 0xc00035a000}, 0x1d3fa20, 0xc0007b9280) /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xbf google.golang.org/grpc.(*Server).processStreamingRPC(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008ead80, 0xc000a4f1d0, 0x38e7b80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1547 +0xe9a google.golang.org/grpc.(*Server).handleStream(0xc0002ec700, {0x26a25b8, 0xc000596000}, 0xc0008ead80, 0x0) /src/loki/vendor/google.golang.org/grpc/server.go:1620 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() /src/loki/vendor/google.golang.org/grpc/server.go:921 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 /src/loki/vendor/google.golang.org/grpc/server.go:919 +0x294 goroutine 1243 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00000a5a0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1573 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:359 +0x18e5 goroutine 1321 [select, 34 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00163a1a0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1125 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:335 +0x1857 goroutine 1244 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07ef880, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0000ab780, 0xc0016be000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0000ab780, {0xc0016be000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0000ab780, {0xc0016be000, 0x1060100000000, 0x8}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc00000e1a8, {0xc0016be000, 0x98f1d0, 0x1}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc000a7e720, {0xc000cfa2e0, 0x9, 0x18}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc000a7e720}, {0xc000cfa2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc000cfa2e0, 0x9, 0x5b06822559f}, {0x2628900, 0xc000a7e720}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc000cfa2a0) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000a5a0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1499 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:364 +0x192f goroutine 1320 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0016345f0, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000c98360) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:326 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:323 +0x180f goroutine 1322 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07ef790, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a3ab80, 0xc00163c000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a3ab80, {0xc00163c000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a3ab80, {0xc00163c000, 0x9a3a6c, 0x800000601}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0008c8060, {0xc00163c000, 0x98a288, 0xc000142390}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc001680c00, {0xc00040cd60, 0x9, 0x101000000000018}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x2628900, 0xc001680c00}, {0xc00040cd60, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00040cd60, 0x9, 0xc000142390}, {0x2628900, 0xc001680c00}) /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00040cd20) /src/loki/vendor/golang.org/x/net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00163a1a0, 0x0, 0x0) /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:605 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc0002ec700, {0x26a25b8, 0xc00163a1a0}) /src/loki/vendor/google.golang.org/grpc/server.go:905 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() /src/loki/vendor/google.golang.org/grpc/server.go:847 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn /src/loki/vendor/google.golang.org/grpc/server.go:846 +0x185 goroutine 1245 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000cfe870, 0x1) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000d00540) /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:414 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_client.go:412 +0x1f85 goroutine 1304 [select, 34 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000c62c80) /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:77 +0xa7 created by google.golang.org/grpc.newCCBalancerWrapper /src/loki/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:67 +0x246 goroutine 1807 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7fbac07ef6a0, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a3bf80, 0xc000d820d1, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a3bf80, {0xc000d820d1, 0x1, 0x1}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a3bf80, {0xc000d820d1, 0xc000c84d80, 0xc0016fc228}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000832138, {0xc000d820d1, 0x5a841e2e56e, 0xc000cd17d0}) /usr/local/go/src/net/net.go:183 +0x45 net/http.(*connReader).backgroundRead(0xc000d820c0) /usr/local/go/src/net/http/server.go:672 +0x3f created by net/http.(*connReader).startBackgroundRead /usr/local/go/src/net/http/server.go:668 +0xcf goroutine 1916 [IO wait]: internal/poll.runtime_pollWait(0x7fbac07ef970, 0x72) /usr/local/go/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000a54680, 0xc000946101, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000a54680, {0xc000946101, 0x1, 0x1}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000a54680, {0xc000946101, 0xc0003e5d40, 0x4}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0008c8068, {0xc000946101, 0x394b9e0, 0xc00091df68}) /usr/local/go/src/net/net.go:183 +0x45 net/http.(*connReader).backgroundRead(0xc0009460f0) /usr/local/go/src/net/http/server.go:672 +0x3f created by net/http.(*connReader).startBackgroundRead /usr/local/go/src/net/http/server.go:668 +0xcf goroutine 1806 [select, 2 minutes]: github.com/grafana/loki/pkg/lokifrontend/frontend/v2.(*Frontend).RoundTripGRPC(0xc000a57cc0, {0x266fb88, 0xc000d82660}, 0xc000a24a00) /src/loki/pkg/lokifrontend/frontend/v2/frontend.go:220 +0x69f github.com/grafana/loki/pkg/lokifrontend/frontend/transport.(*grpcRoundTripperAdapter).RoundTrip(0xc00007ea40, 0xc0000acb00) /src/loki/pkg/lokifrontend/frontend/transport/roundtripper.go:43 +0x96 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.roundTripper.Do({{0x262ace0, 0xc00007ea40}, {0x2631240, 0xc000a53038}, {0x267adb8, 0x394b9e0}, {0x0, 0x0, 0x0}}, {0x266fb88, ...}, ...) /src/loki/pkg/querier/queryrange/queryrangebase/roundtrip.go:174 +0x104 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.retry.Do({{0x2629f40, 0xc0002880a0}, {0x262aec0, 0xc000a59e50}, 0x5, 0xc00030bb00}, {0x266fb88, 0xc000d82660}, {0x26b1f98, 0xc00037f240}) /src/loki/pkg/querier/queryrange/queryrangebase/retry.go:64 +0x1bf github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.InstrumentMiddleware.func1.1.1({0x266fb88, 0xc000d82660}) /src/loki/pkg/querier/queryrange/queryrangebase/instrumentation.go:28 +0x46 github.com/weaveworks/common/instrument.CollectedRequest({0x266fb88, 0xc000d82630}, {0x21dae01, 0x5}, {0x264c590, 0xc0001a0a80}, 0x0, 0xc0017d01d0) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:174 +0x274 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.InstrumentMiddleware.func1.1({0x266fb88, 0xc000d82630}, {0x26b1f98, 0xc00037f240}) /src/loki/pkg/querier/queryrange/queryrangebase/instrumentation.go:26 +0xbb github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.HandlerFunc.Do(0xc00036bc20, {0x266fb88, 0xc000d82630}, {0x26b1f98, 0xc00037f240}) /src/loki/pkg/querier/queryrange/queryrangebase/roundtrip.go:82 +0x3d github.com/grafana/loki/pkg/querier/queryrange.(*splitByInterval).Do(0xc000a7cb40, {0x266fb88, 0xc000d82630}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/split_by_interval.go:193 +0x393 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.InstrumentMiddleware.func1.1.1({0x266fb88, 0xc000d82630}) /src/loki/pkg/querier/queryrange/queryrangebase/instrumentation.go:28 +0x46 github.com/weaveworks/common/instrument.CollectedRequest({0x266fb88, 0xc000d82600}, {0x21f30e0, 0x11}, {0x264c590, 0xc0001a0a78}, 0x0, 0xc0017d0538) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:174 +0x274 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.InstrumentMiddleware.func1.1({0x266fb88, 0xc000d82600}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/queryrangebase/instrumentation.go:26 +0xbb github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.HandlerFunc.Do(0x1c5f2f00, {0x266fb88, 0xc000d82600}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/queryrangebase/roundtrip.go:82 +0x3d github.com/grafana/loki/pkg/querier/queryrange.limitsMiddleware.Do({{0x269f320, 0xc0001253b0}, {0x2631240, 0xc000a7cb80}}, {0x266fb88, 0xc000d824b0}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/limits.go:139 +0x932 github.com/grafana/loki/pkg/querier/queryrange.StatsCollectorMiddleware.func1.1({0x266fb88, 0xc000d824b0}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/stats.go:114 +0xf4 github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.HandlerFunc.Do(0xc00037f180, {0x266fb88, 0xc000d824b0}, {0x26b1f98, 0xc00037f180}) /src/loki/pkg/querier/queryrange/queryrangebase/roundtrip.go:82 +0x3d github.com/grafana/loki/pkg/querier/queryrange/queryrangebase.roundTripper.RoundTrip({{0x262ace0, 0xc00007ea40}, {0x2631240, 0xc000a53038}, {0x267adb8, 0x394b9e0}, {0x0, 0x0, 0x0}}, 0xc0000aca00) /src/loki/pkg/querier/queryrange/queryrangebase/roundtrip.go:155 +0x154 github.com/grafana/loki/pkg/querier/queryrange.roundTripper.RoundTrip({{0x262ace0, 0xc00007ea40}, {0x26311c0, 0xc000a7ca80}, {0x2631280, 0xc000a4f740}, {0x26311c0, 0xc000a7cac0}, {0x26312c0, 0xc000a59ea0}, ...}, ...) /src/loki/pkg/querier/queryrange/roundtrip.go:167 +0xc4 github.com/grafana/loki/pkg/lokifrontend/frontend/transport.(*Handler).ServeHTTP(0xc000a7f200, {0x264a7f0, 0xc00082eb80}, 0xc) /src/loki/pkg/lokifrontend/frontend/transport/handler.go:129 +0x573 github.com/grafana/loki/pkg/util/server.ResponseJSONMiddleware.func1.1({0x264a7f0, 0xc00082eb80}, 0xc0000aca98) /src/loki/pkg/util/server/middleware.go:30 +0xf4 net/http.HandlerFunc.ServeHTTP(0xc0000aca00, {0x264a7f0, 0xc00082eb80}, 0x7fbae765d878) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/util/server.NewPrepopulateMiddleware.func1.1({0x264a7f0, 0xc00082eb80}, 0x25ef301) /src/loki/pkg/util/server/middleware.go:21 +0xa2 net/http.HandlerFunc.ServeHTTP(0x266fb88, {0x264a7f0, 0xc00082eb80}, 0x25ef320) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/querier/queryrange.statsHTTPMiddleware.func1.1({0x26503d0, 0xc000d82390}, 0xc0000ac900) /src/loki/pkg/querier/queryrange/stats.go:90 +0x310 net/http.HandlerFunc.ServeHTTP(0x266fb88, {0x26503d0, 0xc000d82390}, 0x25eb6c8) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/util/fakeauth.glob..func1.1({0x26503d0, 0xc000d82390}, 0xc0000ac800) /src/loki/pkg/util/fakeauth/fake_auth.go:54 +0x294 net/http.HandlerFunc.ServeHTTP(0xc000152460, {0x26503d0, 0xc000d82390}, 0x10000c000ab31c8) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/util/server.glob..func1.1({0x26503d0, 0xc000d82390}, 0x0) /src/loki/pkg/util/server/recovery.go:32 +0x82 net/http.HandlerFunc.ServeHTTP(0xc000152460, {0x26503d0, 0xc000d82390}, 0x0) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/util/httpreq.ExtractQueryTagsMiddleware.func1.1({0x26503d0, 0xc000d82390}, 0xc0000ac800) /src/loki/pkg/util/httpreq/tags.go:34 +0x314 net/http.HandlerFunc.ServeHTTP(0xc0000ac700, {0x26503d0, 0xc000d82390}, 0x2644768) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/gorilla/mux.(*Router).ServeHTTP(0xc0001e0b40, {0x26503d0, 0xc000d82390}, 0xc0000ac600) /src/loki/vendor/github.com/gorilla/mux/mux.go:210 +0x1cf github.com/weaveworks/common/middleware.Instrument.Wrap.func1.2({0x26503d0, 0xc000d82390}) /src/loki/vendor/github.com/weaveworks/common/middleware/instrument.go:70 +0x39 github.com/felixge/httpsnoop.CaptureMetricsFn({0x264c5c0, 0xc000a245f0}, 0xc000ab3600) /src/loki/vendor/github.com/felixge/httpsnoop/capture_metrics.go:76 +0x212 github.com/weaveworks/common/middleware.Instrument.Wrap.func1({0x264c5c0, 0xc000a245f0}, 0xc0000ac600) /src/loki/vendor/github.com/weaveworks/common/middleware/instrument.go:69 +0x35b net/http.HandlerFunc.ServeHTTP(0xc0000ac600, {0x264c5c0, 0xc000a245f0}, 0x0) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/weaveworks/common/middleware.Log.Wrap.func1({0x26507f0, 0xc000a24460}, 0xc0000ac600) /src/loki/vendor/github.com/weaveworks/common/middleware/logging.go:55 +0x287 net/http.HandlerFunc.ServeHTTP(0x40d547, {0x26507f0, 0xc000a24460}, 0xc00036b901) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5({0x264e780, 0xc0004236c0}, 0xc0000ac500) /src/loki/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:154 +0x62d net/http.HandlerFunc.ServeHTTP(0x266fae0, {0x264e780, 0xc0004236c0}, 0x25eb6c8) /usr/local/go/src/net/http/server.go:2047 +0x2f github.com/grafana/loki/pkg/loki.(*Loki).initServer.func2.1({0x264e780, 0xc0004236c0}, 0xc0000ac400) /src/loki/pkg/loki/modules.go:127 +0x2c4 net/http.HandlerFunc.ServeHTTP(0x0, {0x264e780, 0xc0004236c0}, 0x46928e) /usr/local/go/src/net/http/server.go:2047 +0x2f net/http.serverHandler.ServeHTTP({0xc000d820c0}, {0x264e780, 0xc0004236c0}, 0xc0000ac400) /usr/local/go/src/net/http/server.go:2879 +0x43b net/http.(*conn).serve(0xc0003d0aa0, {0x266fb88, 0xc000a0cf60}) /usr/local/go/src/net/http/server.go:1930 +0xb08 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:3034 +0x4e8