$ $GOPATH/bin/pp -aggressive deadlock.cleaned.stacktrace SIGQUIT: quit PC=0x45ebe3 m=0 sigcode=0 rax 0xfffffffffffffffc rbx 0xfed70b8 rcx 0xffffffffffffffff rdx 0x0 rdi 0x1af9020 rsi 0x80 rbp 0xc0003826e8 rsp 0xc0003826a0 r8 0x0 r9 0x0 r10 0xc0003826d8 r11 0x206 r12 0x7aa716105f55f3 r13 0xe6 r14 0x3 r15 0xc0019a35c0 rip 0x45ebe3 rflags 0x206 cs 0x33 fs 0x0 gs 0x0 1: syscall [Created by runtime.(*timersBucket).addtimerLocked @ time.go:169] runtime lock_futex.go:227 notetsleepg(*note(0x1af9020), 267219128, 0x0) runtime time.go:311 timerproc(*timersBucket(0x1af9000)) runtime asm_amd64.s:1337 goexit() 10: semacquire [90~91 minutes] [Created by http.(*Server).Serve @ server.go:2884] sync sema.go:71 runtime_SemacquireMutex(#8471, #2579) sync rwmutex.go:50 (*RWMutex).RLock(...) memory meta_tags.go:816 (*metaTagEnricher).countMetricsWithMetaTags(#8470, 0) memory memory.go:1130 (*UnpartitionedMemoryIdx).FindTagValuesWithQuery(#5692, 1, *, *, 0, 0, 0, *, *, *, ...) api cluster.go:249 (*Server).indexAutoCompleteTagValues(#9920, *, 1, *, *, 0, 0, *, *, 4, ...) reflect value.go:447 Value.call(#46, #9902, 0x13, #68, 4, *, 2, 2, #66, *, ...) reflect value.go:308 Value.Call(#46, #9902, 0x13, *, 2, 2, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #46, #9902, #86, #46, 2, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #46, #9902, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, #1, *) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, *, *, 1, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, *, *, 0x16, #57) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware logger.go:45 Logger.func1(*) reflect value.go:447 Value.call(#44, #71, 0x13, #68, 4, *, 1, 1, #66, *, ...) reflect value.go:308 Value.Call(#44, #71, 0x13, *, 1, 1, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #44, #71, #86, #44, 1, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #44, #71, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware tracer.go:70 Tracer.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9656, *, 1, 1, #67, *, #50129, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9656, #86, #51, 1, *, *, *, *, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9656, #12, *, *, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware stats.go:76 RequestStats.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9900, *, 1, 1, #67, *, #50129, 0, 0) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9900, #86, #51, 1, 0, 0, 0, 0, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9900, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, *, *, 0xc0, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, #9681, 0x30, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 router.go:187 (*Router).Handle.func1(#80, *, *, 0) macaron.v1 router.go:294 (*Router).ServeHTTP(#11806, #80, *, *) macaron.v1 macaron.go:220 (*Macaron).ServeHTTP(#5861, #80, *, *) http server.go:2774 serverHandler.ServeHTTP(#2558, #80, *, *) http server.go:1878 (*conn).serve(*, #82, *) 1283: semacquire [89~91 minutes] [Created by http.(*Server).Serve @ server.go:2884] sync sema.go:71 runtime_SemacquireMutex(#8471, #2579) sync rwmutex.go:50 (*RWMutex).RLock(...) memory meta_tags.go:816 (*metaTagEnricher).countMetricsWithMetaTags(#8470, 0) memory memory.go:762 (*UnpartitionedMemoryIdx).FindByTag(#5692, 1, *, *, *, *, 0xffffffffffffffff, *, *, 0xffffffffffffffff) api cluster.go:332 (*Server).indexFindByTag(#9920, *, 1, *, *, *, *) reflect value.go:447 Value.call(#47, #9901, 0x13, #68, 4, *, 2, 2, #66, *, ...) reflect value.go:308 Value.Call(#47, #9901, 0x13, *, 2, 2, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #47, #9901, #86, #47, 2, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #47, #9901, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, #1, *) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, *, *, 1, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, *, *, 0x16, #57) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware logger.go:45 Logger.func1(*) reflect value.go:447 Value.call(#44, #71, 0x13, #68, 4, *, 1, 1, #66, *, ...) reflect value.go:308 Value.Call(#44, #71, 0x13, *, 1, 1, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #44, #71, #86, #44, 1, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #44, #71, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware tracer.go:70 Tracer.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9656, *, 1, 1, #67, *, #50129, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9656, #86, #51, 1, *, *, *, *, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9656, #12, *, *, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware stats.go:76 RequestStats.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9900, *, 1, 1, #67, *, #50129, 0, 0) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9900, #86, #51, 1, 0, 0, 0, 0, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9900, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, *, *, *, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, #9655, 0x30, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 router.go:187 (*Router).Handle.func1(#80, *, *, 0) macaron.v1 router.go:294 (*Router).ServeHTTP(#11806, #80, *, *) macaron.v1 macaron.go:220 (*Macaron).ServeHTTP(#5861, #80, *, *) http server.go:2774 serverHandler.ServeHTTP(#2558, #80, *, *) http server.go:1878 (*conn).serve(*, #82, *) 237: semacquire [91 minutes] [Created by http.(*Server).Serve @ server.go:2884] sync sema.go:71 runtime_SemacquireMutex(#5694, 0) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:769 (*UnpartitionedMemoryIdx).FindByTag(#5692, 1, *, *, *, *, 0xffffffffffffffff, *, *, 0xffffffffffffffff) api cluster.go:332 (*Server).indexFindByTag(#9920, *, 1, *, *, *, *) reflect value.go:447 Value.call(#47, #9901, 0x13, #68, 4, *, 2, 2, #66, #73, ...) reflect value.go:308 Value.Call(#47, #9901, 0x13, *, 2, 2, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #47, #9901, #86, #47, 2, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #47, #9901, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, #1, 8) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, *, *, 1, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, *, *, 0x16, #57) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware logger.go:45 Logger.func1(*) reflect value.go:447 Value.call(#44, #71, 0x13, #68, 4, *, 1, 1, *, #73, ...) reflect value.go:308 Value.Call(#44, #71, 0x13, *, 1, 1, 0, 0, 0) inject inject.go:177 (*injector).callInvoke(*, #44, #71, #86, #44, 1, 0, 0, 0, 0, ...) inject inject.go:137 (*injector).Invoke(*, #44, #71, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware tracer.go:70 Tracer.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9656, *, 1, 1, #67, *, #50129, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9656, #86, #51, 1, *, *, *, *, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9656, #12, *, *, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) middleware stats.go:76 RequestStats.func1(*) macaron.v1 context.go:79 ContextInvoker.Invoke(#9900, *, 1, 1, #67, *, #50129, 0, 0) inject inject.go:157 (*injector).fastInvoke(*, #50129, #9900, #86, #51, 1, 0, 0, 0, 0, ...) inject inject.go:135 (*injector).Invoke(*, #51, #9900, 0, 0, 0, 0, 0) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 context.go:112 (*Context).Next(...) macaron.v1 recovery.go:161 Recovery.func1(*, #6006) macaron.v1 logger.go:40 LoggerInvoker.Invoke(#72, *, 2, 2, #61, #6006, #50128, *, *) inject inject.go:157 (*injector).fastInvoke(*, #50128, #72, #86, #52, 2, *, #6, *, 0xc0, ...) inject inject.go:135 (*injector).Invoke(*, #52, #72, *, #9655, 0x30, *, *) macaron.v1 context.go:121 (*Context).run(*) macaron.v1 router.go:187 (*Router).Handle.func1(#80, *, *, 0) macaron.v1 router.go:294 (*Router).ServeHTTP(#11806, #80, *, *) macaron.v1 macaron.go:220 (*Macaron).ServeHTTP(#5861, #80, *, *) http server.go:2774 serverHandler.ServeHTTP(#2558, #80, *, *) http server.go:1878 (*conn).serve(*, #82, *) 15: semacquire [91 minutes] [Created by kafkamdm.(*KafkaMdm).Start @ kafkamdm.go:231] sync sema.go:71 runtime_SemacquireMutex(#5694, *) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:373 (*UnpartitionedMemoryIdx).Update(#5692, *, *, 1, *, *, *, 0, 0, 0, ...) cassandra cassandra.go:210 (*CasIdx).Update(#5695, *, *, 1, *, *, *, 0, 0, 0, ...) input input.go:100 DefaultHandler.ProcessMetricPoint(#20455, #20456, #20457, #20547, #20548, #20549, #20458, #77, #4247, #85, ...) kafkamdm kafkamdm.go:340 (*KafkaMdm).handleMsg(#6199, *, 0x21, *, *) kafkamdm kafkamdm.go:321 (*KafkaMdm).consumePartition(#6199, #2051, 0xa, *, *) 2: semacquire [91 minutes] [Created by kafkamdm.(*KafkaMdm).Start @ kafkamdm.go:231] sync sema.go:71 runtime_SemacquireMutex(#5694, *) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:422 (*UnpartitionedMemoryIdx).AddOrUpdate(#5692, *, *, *, *, *, 0, 0, 0, 0, ...) cassandra cassandra.go:239 (*CasIdx).AddOrUpdate(#5695, *, *, *, *, *, 0, 0, 0, 0, ...) input input.go:176 DefaultHandler.ProcessMetricData(#20455, #20456, #20457, #20547, #20548, #20549, #20458, #77, #4247, #85, ...) kafkamdm kafkamdm.go:352 (*KafkaMdm).handleMsg(#6199, *, *, *, *) kafkamdm kafkamdm.go:321 (*KafkaMdm).consumePartition(#6199, #2051, 0xa, *, *) 1: semacquire [91 minutes] [Created by memory.NewWriteQueue @ write_queue.go:36] sync sema.go:71 runtime_SemacquireMutex(*uint32(#5693), bool(#31896)) sync rwmutex.go:98 (*RWMutex).Lock(*RWMutex(#5692)) memory memory.go:517 (*UnpartitionedMemoryIdx).indexTags(*UnpartitionedMemoryIdx(#5692), *MetricDefinition(#25554)) memory memory.go:630 (*UnpartitionedMemoryIdx).add(*UnpartitionedMemoryIdx(#5692), *Archive(#25554)) memory write_queue.go:76 (*WriteQueue).flush(*WriteQueue(#6007)) memory write_queue.go:95 (*WriteQueue).loop(#6007) 1: semacquire [Created by cassandra.(*CasIdx).Init @ cassandra.go:185] sync sema.go:71 runtime_SemacquireMutex(*uint32(#5694), bool(#11111)) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:1640 (*UnpartitionedMemoryIdx).Prune(*UnpartitionedMemoryIdx(#5692), Time(0xbfaa09390d05ad07), #96, 0x2, 0x0, #37267, #11112, #11078) cassandra cassandra.go:668 (*CasIdx).Prune(*CasIdx(#5695), Time(0xbfaa09390d05ad07), #96, #49836, 0x2322, 0x2328, 0x0, 0x0) cassandra cassandra.go:685 (*CasIdx).prune(#5695) 1: semacquire [91 minutes] [Created by memory.(*metaTagEnricher).delMetaRecord @ meta_tags.go:780] sync sema.go:56 runtime_Semacquire(*uint32(#49786)) sync waitgroup.go:130 (*WaitGroup).Wait(*WaitGroup(#49785)) memory tag_query.go:225 (*TagQueryContext).Run(*TagQueryContext(#49785), TagIndex(#13183), map[MKey]*Archive(0x0), *metaTagHierarchy(#45604)) memory memory.go:311 NewUnpartitionedMemoryIdx.func1(0x1, 0x0, #2899, 0x1, 0x1, 0xffffffffffffffff, #45604) memory meta_tags.go:91 (*metaTagIdx).getOrgMetaTagIndex.func1(0, #2899, 1, 1, 0xffffffffffffffff, #45604) 1: runnable [Created by gocql.(*Conn).init @ conn.go:283] syscall asm_linux_amd64.s:18 Syscall(0, 0x2c2, #8867, 0x1000, 0xffffffffffffffff, 0, 0xb) syscall zsyscall_linux_amd64.go:732 read(706, []byte(#8867 len=4096 cap=4096), 0x0, 0x60, 0x0) syscall syscall_unix.go:172 Read(...) poll fd_unix.go:165 (*FD).Read(*FD(#8438), []byte(#8867 len=4096 cap=4096), 0x0, 0x0, 0x0) net fd_unix.go:202 (*netFD).Read(*netFD(#8438), []byte(#8867 len=4096 cap=4096), #9, #8818, 0x0) net net.go:177 (*conn).Read(*conn(#3940), #8867, 0x1000, 0x1000, 0x0, 0x0, 0x0) bufio bufio.go:223 (*Reader).Read(*Reader(#8462), []byte(#7971 len=1 cap=9), 0x2, 0x3, 0x60) io io.go:310 ReadAtLeast(Reader(#74), []byte(#7971 len=1 cap=9), 1, 0x38, #37391, 0x0) io io.go:329 ReadFull(...) gocql frame.go:449 readHeader(Reader(#74), []byte(#7971 len=9 cap=9), 0x0, 0x0, 0x0, 0x0, 0x0, ...) gocql conn.go:601 (*Conn).recv(*Conn(#7970), Context(#82), 0x0, 0x0) gocql conn.go:518 (*Conn).serve(#7970, #82, #8204) 1302: IO wait [Created by gocql.(*Conn).init @ conn.go:283] poll netpoll.go:182 runtime_pollWait(*, 0x72, 0xffffffffffffffff) poll fd_poll_runtime.go:87 (*pollDesc).wait(*, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:169 (*FD).Read(*, *, 0x1000, 0x1000, 0, 0, 0) net fd_unix.go:202 (*netFD).Read(*, *, 0x1000, 0x1000, *, *, *) net net.go:177 (*conn).Read(*, *, 0x1000, 0x1000, 0, 0, 0) bufio bufio.go:223 (*Reader).Read(*, *, 1, 9, *, *, *) io io.go:310 ReadAtLeast(#74, *, *, 1, 9, 1, 0x38, *, 0) io io.go:329 ReadFull(...) gocql frame.go:449 readHeader(#74, *, *, 9, 9, 0, 0, 0, 0, 0, ...) gocql conn.go:601 (*Conn).recv(*, #82, *, 0, 0) gocql conn.go:518 (*Conn).serve(*, #82, *) 1: chan receive [91 minutes] [Created by memory.(*metaTagEnricher).start @ meta_tags.go:539] memory meta_tags.go:748 (*metaTagEnricher)._addMetaRecord(*metaTagEnricher(#8470), interface{}(#56)) memory meta_tags.go:573 (*metaTagEnricher).start.func1(#8470) 17: select [91 minutes] [Created by sarama.(*consumer).ConsumePartition @ consumer.go:158] sarama consumer.go:467 (*partitionConsumer).responseFeeder(*) sarama utils.go:45 withRecover(*) 17: chan receive [91 minutes] [Created by sarama.(*consumer).newBrokerConsumer @ consumer.go:719] sarama consumer.go:774 (*brokerConsumer).subscriptionConsumer(*) sarama utils.go:45 withRecover(*) 17: chan receive [1080 minutes] [Created by sarama.(*consumer).ConsumePartition @ consumer.go:157] sarama consumer.go:337 (*partitionConsumer).dispatcher(*) sarama utils.go:45 withRecover(*) 1: select [5 minutes] [Created by sarama.NewClient @ client.go:178] sarama client.go:755 (*client).backgroundMetadataUpdater(*client(#5759)) sarama utils.go:45 withRecover(#2835) 17: select [91 minutes] [Created by sarama.(*consumer).newBrokerConsumer @ consumer.go:718] sarama consumer.go:745 (*brokerConsumer).subscriptionManager(*) sarama utils.go:45 withRecover(*) 1: runnable [Created by gocql.(*Conn).init @ conn.go:284] gocql conn.go:910 (*Conn).exec(*Conn(#7970), Context(#83), frameWriter(#75), Tracer(0x0), 0x0, 0x0, 0x0) gocql conn.go:564 (*Conn).heartBeat(#7970, #82, #8204) 1: select [Created by gocql.(*Conn).init @ conn.go:284] gocql conn.go:910 (*Conn).exec(*Conn(#12075), Context(#83), frameWriter(#75), Tracer(0x0), 0x0, 0x0, 0x0) gocql conn.go:564 (*Conn).heartBeat(#12075, #82, #9710) 18: chan receive [0~5 minutes] [Created by sarama.(*Broker).Open.func1 @ broker.go:218] sarama broker.go:823 (*Broker).responseReceiver(*) sarama utils.go:45 withRecover(*) 1: semacquire [91 minutes] [Created by memory.(*TagQueryContext).Run @ tag_query.go:219] sync sema.go:56 runtime_Semacquire(*uint32(#33148)) sync waitgroup.go:130 (*WaitGroup).Wait(*WaitGroup(#33147)) memory tag_query_id_selector.go:73 (*idSelector).getIds(*idSelector(#33146), chan MKey(#45604)) memory tag_query.go:221 (*TagQueryContext).Run.func2(#49785, #45604) 17: semacquire [91 minutes] [Created by memory.(*metaTagEnricher).addMetaRecord @ meta_tags.go:734] sync sema.go:71 runtime_SemacquireMutex(#5694, *) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:301 NewUnpartitionedMemoryIdx.func1(*, 0, *, 1, 1, 0xffffffffffffffff, *) memory meta_tags.go:91 (*metaTagIdx).getOrgMetaTagIndex.func1(0, *, 1, 1, 0xffffffffffffffff, *) 18: semacquire [91 minutes] [Created by memory.(*metaTagEnricher).delMetaRecord @ meta_tags.go:780] sync sema.go:71 runtime_SemacquireMutex(#5694, *) sync rwmutex.go:50 (*RWMutex).RLock(...) memory memory.go:301 NewUnpartitionedMemoryIdx.func1(1, 0, *, 1, 1, 0xffffffffffffffff, *) memory meta_tags.go:91 (*metaTagIdx).getOrgMetaTagIndex.func1(0, *, 1, 1, 0xffffffffffffffff, *) 1: IO wait [Created by main.main @ metrictank.go:425] poll netpoll.go:182 runtime_pollWait(*pollDesc(#50721), 114, 0x0) poll fd_poll_runtime.go:87 (*pollDesc).wait(*pollDesc(#8672), 114, bool(0x0), #69) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:384 (*FD).Accept(*FD(#8671), 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) net fd_unix.go:238 (*netFD).accept(*netFD(#8671), #2055, #2054, #4) net tcpsock_posix.go:139 (*TCPListener).accept(*TCPListener(#3899), #23790, #1884, #55781) net tcpsock.go:247 (*TCPListener).AcceptTCP(*TCPListener(#3899), #23791, #17, #1883) api api.go:158 tcpKeepAliveListener.Accept(#3899, #23792, 0x18, #9470, #23) http server.go:2859 (*Server).Serve(*Server(#2558), #81, #3899, 0x0, 0x0) api api.go:135 (*Server).Run(#9920) 1: select [Created by go-hostpool.NewEpsilonGreedy @ epsilon_greedy.go:65] go-hostpool epsilon_greedy.go:94 (*epsilonGreedyHostPool).epsilonGreedyDecay(#3310) 1: select [Created by memory.NewFindCache @ find_cache.go:78] memory find_cache.go:294 (*FindCache).processInvalidateQueue(#2580) 1: select [1085 minutes] [Created by cache.NewCCache @ ccache.go:66] cache ccache.go:79 (*CCache).evictLoop(#4361) 1: select [Created by kafkamdm.(*KafkaMdm).MaintainPriority @ kafkamdm.go:392] kafkamdm kafkamdm.go:395 (*KafkaMdm).MaintainPriority.func1(#6199) 3: select [Created by gocql.(*Session).init @ session.go:257] gocql session.go:291 (*Session).reconnectDownedHosts(*, #1889) 1: select [Created by cassandra.(*MetaRecordIdx).Start @ meta_record_idx.go:102] cassandra meta_record_idx.go:120 (*MetaRecordIdx).pollStore(#6203) 1: select [Created by memory.NewFindCache @ find_cache.go:79] memory find_cache.go:222 (*FindCache).stats(#2580) 1: select [1085 minutes] [Created by cassandra.(*MetaRecordIdx).Start @ meta_record_idx.go:105] cassandra meta_record_idx.go:195 (*MetaRecordIdx).pruneMetaRecords(#6203) 4: runnable [Created by gocql.(*Conn).init @ conn.go:284] gocql conn.go:558 (*Conn).heartBeat(*, #82, *) 1: select [1080 minutes] main metrictank.go:501 main() 1: chan receive [Created by stats.NewGraphite @ out_graphite.go:64] stats out_graphite.go:95 (*Graphite).reporter(#2210, 0x1e) 1: chan receive [Created by main.main @ metrictank.go:226] heap heap.go:49 Heap.Run(#2455, 4, #1885, 0xe10, #1889, #1882, #2560) 1: select [1 minutes] [Created by memberlist.newMemberlist @ memberlist.go:178] memberlist net.go:196 (*Memberlist).streamListen(#3285) 20: select [89~90 minutes] [Created by cassandra.NewCassandraStore @ cassandra.go:230] cassandra cassandra.go:410 (*CassandraStore).processReadQueue(#5758) 5: select [Created by cassandra.NewCassandraStore @ cassandra.go:225] cassandra cassandra.go:299 (*CassandraStore).processWriteQueue(#5758, *, *) 1: select [Created by memberlist.(*Memberlist).schedule @ state.go:99] memberlist state.go:133 (*Memberlist).triggerFunc(#3285, #1593, #6201, #4915, #2836) 1: select [Created by memberlist.(*Memberlist).schedule @ state.go:111] memberlist state.go:133 (*Memberlist).triggerFunc(#3285, #352, #6202, #4915, #2837) 1: select [Created by memberlist.newMemberlist @ memberlist.go:180] memberlist net.go:376 (*Memberlist).packetHandler(#3285) 3: select [Created by gocql.(*controlConn).connect @ control.go:259] gocql control.go:70 (*controlConn).heartBeat(*) 6: select [84~1085 minutes] [Created by gocql.newEventDebouncer @ events.go:27] gocql events.go:39 (*eventDebouncer).flusher(*) 1: chan receive [1085 minutes] [Created by api.(*Server).Run @ api.go:117] api api.go:148 (*Server).handleShutdown(#9920, #79, #3899) 1: select [1 minutes] [Created by memberlist.(*Memberlist).schedule @ state.go:105] memberlist state.go:160 (*Memberlist).pushPullTrigger(#3285, #4915) 3: select [Created by cassandra.NewSession @ cassandra_session.go:54] cassandra cassandra_session.go:123 (*Session).deadConnectionRefresh(*) 1: select [Created by memberlist.newMemberlist @ memberlist.go:179] memberlist net.go:289 (*Memberlist).packetListen(#3285) 1297: select [Created by gocql.(*Conn).init @ conn.go:284] gocql conn.go:558 (*Conn).heartBeat(*, #82, *) 1: select [91 minutes] [Created by memory.(*idSelector).byTagValue @ tag_query_id_selector.go:105] memory tag_query_id_selector.go:129 (*idSelector).byTagValueFromMetricTagIndex(#33146) 1: select [Created by cluster.(*MemberlistManager).BroadcastUpdate @ manager.go:350] memberlist memberlist.go:457 (*Memberlist).UpdateNode(#3285, #1593, 0, #13439) 1: chan receive [1085 minutes] [Created by main.main @ metrictank.go:221] main metrictank.go:222 main.func1(#2560) 1: select [Created by view.init.0 @ worker.go:29] view worker.go:150 (*worker).start(#2319) 1: chan receive [Created by go-metrics.NewMeter @ meter.go:46] go-metrics meter.go:245 (*meterArbiter).tick(#95) 1: select [Created by jaeger-client-go.NewRemoteReporter @ reporter.go:209] jaeger-client-go reporter.go:265 (*remoteReporter).processQueue(#2209) 1300: select [Created by gocql.newWriteCoalescer @ conn.go:727] gocql conn.go:822 (*writeCoalescer).writeFlusher(*, 0x30d40) 1: chan receive [89 minutes] [Created by accnt.NewFlatAccnt @ flat_accnt.go:135] accnt flat_accnt.go:190 (*FlatAccnt).eventLoop(#4221) 1: chan receive [Created by stats.NewGraphite @ out_graphite.go:62] stats out_graphite.go:122 (*Graphite).writer(#2210) 1: sleep [45 minutes] [Created by mdata.NewAggMetrics @ aggmetrics.go:44] runtime proc.go:307 goparkunlock(...) time time.go:105 Sleep(3593799546976) mdata aggmetrics.go:56 (*AggMetrics).GC(#4247) 17: sleep [Created by kafkamdm.(*KafkaMdm).consumePartition @ kafkamdm.go:299] runtime proc.go:307 goparkunlock(...) time time.go:105 Sleep(#1593) kafkamdm kafkamdm.go:372 (*KafkaMdm).trackStats(#6199, #2051, 0xa, *) 1: sleep [Created by cluster.(*MemberlistManager).Start @ manager.go:141] runtime proc.go:307 goparkunlock(...) time time.go:105 Sleep(51000000000) cluster manager.go:151 (*MemberlistManager).Start.func1(#2784, 0x3c, 0x2e) 1: sleep [Created by clock.AlignedTickLossy @ clock.go:17] runtime proc.go:307 goparkunlock(...) time time.go:105 Sleep(29999899091) clock clock.go:23 AlignedTickLossy.func1(#1887, #2561) 1: IO wait [91 minutes] [Created by stats.(*Graphite).writer.func1 @ out_graphite.go:114] poll netpoll.go:182 runtime_pollWait(*pollDesc(#50054), 114, 0xffffffffffffffff) poll fd_poll_runtime.go:87 (*pollDesc).wait(*pollDesc(#37001), 114, bool(0x400), 0xffffffffffffffff) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:169 (*FD).Read(*FD(#37000), []byte(#49431 len=1024 cap=1024), 0x0, 0x0, 0x0) net fd_unix.go:202 (*netFD).Read(*netFD(#37000), []byte(#49431 len=1024 cap=1024), #3841, #11, 0x400) net net.go:177 (*conn).Read(*conn(#48789), #49431, 0x400, 0x400, 0x0, 0x0, 0x0) stats out_graphite.go:151 (*Graphite).checkEOF(#2210, #84, #48789, #3796) 1: IO wait [1 minutes] [Created by memberlist.NewNetTransport @ net_transport.go:109] poll netpoll.go:182 runtime_pollWait(*pollDesc(#50906), 114, 0x0) poll fd_poll_runtime.go:87 (*pollDesc).wait(*pollDesc(#5092), 114, bool(0x0), #69) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:384 (*FD).Accept(*FD(#5091), 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) net fd_unix.go:238 (*netFD).accept(*netFD(#5091), #41086, 0x0, 0x0) net tcpsock_posix.go:139 (*TCPListener).accept(*TCPListener(#2365), #4914, #13205, 0x1) net tcpsock.go:247 (*TCPListener).AcceptTCP(*TCPListener(#2365), #13205, 0x0, 0x0) memberlist net_transport.go:225 (*NetTransport).tcpListen(#2584, #2365) 1: IO wait [Created by memberlist.NewNetTransport @ net_transport.go:110] poll netpoll.go:182 runtime_pollWait(*pollDesc(#50905), 114, 0x0) poll fd_poll_runtime.go:87 (*pollDesc).wait(*pollDesc(#5096), 114, bool(0x10000), 0x0) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:219 (*FD).ReadFrom(*FD(#5095), []byte(#17632 len=65536 cap=65536), 0x0, 0x0, 0x0, 0x0, 0x0) net fd_unix.go:208 (*netFD).readFrom(*netFD(#5095), []byte(#17632 len=65536 cap=65536), 0x4, #4, #12980, #12981, #3) net udpsock_posix.go:47 (*UDPConn).readFrom(*UDPConn(#2366), []byte(#17632 len=65536 cap=65536), 0x101, #12979, #12982, #11) net udpsock.go:121 (*UDPConn).ReadFrom(*UDPConn(#2366), []byte(#17632 len=65536 cap=65536), #1594, #78, #43649, 0x0, 0x0) memberlist net_transport.go:247 (*NetTransport).udpListen(#2584, #2366) 1: IO wait [1080 minutes] [Created by carbon.(*Carbon).Start @ carbon.go:120] poll netpoll.go:182 runtime_pollWait(*pollDesc(#51106), 114, 0x0) poll fd_poll_runtime.go:87 (*pollDesc).wait(*pollDesc(#16930), 114, bool(0x0), #69) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:384 (*FD).Accept(*FD(#16929), 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) net fd_unix.go:238 (*netFD).accept(*netFD(#16929), 0x0, 0x0, 0x0) net tcpsock_posix.go:139 (*TCPListener).accept(*TCPListener(#21303), #16842, #14, #9473) net tcpsock.go:247 (*TCPListener).AcceptTCP(*TCPListener(#21303), 0x0, #16843, #19525) carbon carbon.go:136 (*Carbon).accept(#4248) 1: syscall [1085 minutes] [Created by signal.init.0 @ signal_unix.go:29] signal sigqueue.go:139 signal_recv(0x0) signal signal_unix.go:23 loop() 4: IO wait [89 minutes] [Created by http.(*Server).Serve @ server.go:2884] poll netpoll.go:182 runtime_pollWait(*, 0x72, 0xffffffffffffffff) poll fd_poll_runtime.go:87 (*pollDesc).wait(*, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) poll fd_poll_runtime.go:92 (*pollDesc).waitRead(...) poll fd_unix.go:169 (*FD).Read(*, *, 0x1000, 0x1000, 0, 0, 0) net fd_unix.go:202 (*netFD).Read(*, *, 0x1000, 0x1000, *, *, #22) net net.go:177 (*conn).Read(*, *, 0x1000, 0x1000, 0, 0, 0) http server.go:787 (*connReader).Read(*, *, 0x1000, 0x1000, 0, 0, 0x72) bufio bufio.go:100 (*Reader).fill(*) bufio bufio.go:138 (*Reader).Peek(*, 4, 0, 0, 0, 0, #18) http server.go:964 (*conn).readRequest(*, #82, *, 0, 0, 0) http server.go:1819 (*conn).serve(*, #82, *)