添加goroutine跟踪。一个连接（顶部一个）卡在netpoll内部写入。这种联系是我们自己的服务之间的联系，所以不要指望它很慢。可以通过添加写入超时来修复该问题，但可能导致它的原因？
goroutine profile: total 32685
1 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a086 0x62d084 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x5fbc12 0x5f8dd0 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a086 net.(*pollDesc).WaitWrite+0x36 /usr/local/go/src/net/fd_poll_runtime.go:82
# 0x62d084 net.(*netFD).Write+0x414 /usr/local/go/src/net/fd_unix.go:336
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x5fbc12 github.com/moonfrog/heimdall/client.(*Client).sendToService+0x482 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:452
# 0x5f8dd0 github.com/moonfrog/heimdall/client.(*Client).process+0xfd0 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:256
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x460ae1
1148 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x5f1ea3 0x491dac 0x491c99 0x5fa486 0x5f7467 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x5f1ea3 github.com/moonfrog/heimdall/serviceController.(*Controller).OnDisconnect+0x393 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:210
# 0x491dac github.com/moonfrog/heimdall/router.(*Router).onDisconnect+0x8c /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:151
# 0x491c99 github.com/moonfrog/heimdall/router.(*Router).UnregisterClient+0x309 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:146
# 0x5fa486 github.com/moonfrog/heimdall/client.(*Client).disconnect+0x206 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:329
# 0x5f7467 github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x3a7 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:150
11385 @ 0x42fd33 0x43e9f7 0x43df52 0x5fc6df 0x49181f 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x5fc6df github.com/moonfrog/heimdall/client.(*Client).ForceDisconnect+0xdf /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:502
# 0x49181f github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x7af /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:114
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c6d64 0x6f9081 0x704b71 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c6d64 sync.(*WaitGroup).Wait+0xb4 /usr/local/go/src/sync/waitgroup.go:127
# 0x6f9081 github.com/samuel/go-zookeeper/zk.(*Conn).loop+0x8c1 /home/rts/gopath/src/github.com/samuel/go-zookeeper/zk/conn.go:436
# 0x704b71 github.com/samuel/go-zookeeper/zk.Connect.func1+0x21 /home/rts/gopath/src/github.com/samuel/go-zookeeper/zk/conn.go:218
11343 @ 0x42fd33 0x43e9f7 0x43df52 0x5f70ab 0x460ae1
# 0x5f70ab github.com/moonfrog/heimdall/client.(*Client).readPump+0x70b /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:111
4332 @ 0x42fd33 0x43e9f7 0x43df52 0x5f6d54 0x460ae1
# 0x5f6d54 github.com/moonfrog/heimdall/client.(*Client).readPump+0x3b4 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:104
1438 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73ca92 0x7447ee 0x5f1fd4 0x491897 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73ca92 github.com/moonfrog/heimdall/service.(*Service).OnReconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:171
# 0x7447ee github.com/moonfrog/heimdall/service/manager.(*Sticky).Reconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:105
# 0x5f1fd4 github.com/moonfrog/heimdall/serviceController.(*Controller).OnReconnect+0xd4 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:218
# 0x491897 github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x827 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:116
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
18 @ 0x42fd33 0x43e9f7 0x43df52 0x73fbab 0x460ae1
# 0x73fbab github.com/moonfrog/heimdall/service.(*Service).startHB.func1+0x23b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:241
1 @ 0x42fd33 0x42fdf4 0x4061ef 0x405d0b 0x587454 0x460ae1
# 0x587454 github.com/moonfrog/badger/lib/stats/serverside/golang.(*Client).startScheduler+0x94 /home/rts/gopath/src/github.com/moonfrog/badger/lib/stats/serverside/golang/client.go:182
1 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62dcbc 0x64bbfd 0x54d121 0x54c2b9 0x54c106 0x54c808 0x5807ac 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62dcbc net.(*netFD).accept+0x27c /usr/local/go/src/net/fd_unix.go:426
# 0x64bbfd net.(*TCPListener).AcceptTCP+0x4d /usr/local/go/src/net/tcpsock_posix.go:254
# 0x54d121 net/http.tcpKeepAliveListener.Accept+0x41 /usr/local/go/src/net/http/server.go:2427
# 0x54c2b9 net/http.(*Server).Serve+0x129 /usr/local/go/src/net/http/server.go:2117
# 0x54c106 net/http.(*Server).ListenAndServe+0x136 /usr/local/go/src/net/http/server.go:2098
# 0x54c808 net/http.ListenAndServe+0x98 /usr/local/go/src/net/http/server.go:2195
# 0x5807ac github.com/moonfrog/heimdall/debugger.(*Debugger).Start+0x55c /home/rts/gopath/src/github.com/moonfrog/heimdall/debugger/debugger.go:89
2361 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c65b8 0x5f2066 0x5f9d26 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c65b8 sync.(*RWMutex).RLock+0x58 /usr/local/go/src/sync/rwmutex.go:37
# 0x5f2066 github.com/moonfrog/heimdall/serviceController.(*Controller).OnConnect+0x46 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:224
# 0x5f9d26 github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0xe86 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:284
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
387 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x5fbc12 0x5f8dd0 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x5fbc12 github.com/moonfrog/heimdall/client.(*Client).sendToService+0x482 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:452
# 0x5f8dd0 github.com/moonfrog/heimdall/client.(*Client).process+0xfd0 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:256
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c67b1 0x5f3391 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c67b1 sync.(*RWMutex).Lock+0xa1 /usr/local/go/src/sync/rwmutex.go:88
# 0x5f3391 github.com/moonfrog/heimdall/serviceController.(*Controller).monitor+0xc1 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:286
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73fb4c 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73fb4c github.com/moonfrog/heimdall/service.(*Service).startHB.func1+0x1dc /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:252
1 @ 0x42fd33 0x43e9f7 0x43df52 0x4558a8 0x460ae1
# 0x42fd33 runtime.gopark+0x163 /usr/local/go/src/runtime/proc.go:263
# 0x43e9f7 runtime.selectgoImpl+0xa67 /usr/local/go/src/runtime/select.go:392
# 0x43df52 runtime.selectgo+0x12 /usr/local/go/src/runtime/select.go:215
# 0x4558a8 runtime.ensureSigM.func1+0x358 /usr/local/go/src/runtime/signal1_unix.go:279
2 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62dcbc 0x64bbfd 0x5e6f16 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62dcbc net.(*netFD).accept+0x27c /usr/local/go/src/net/fd_unix.go:426
# 0x64bbfd net.(*TCPListener).AcceptTCP+0x4d /usr/local/go/src/net/tcpsock_posix.go:254
# 0x5e6f16 github.com/moonfrog/heimdall/network.(*TCPServer).Start.func1+0x46 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/tcpServer.go:22
12 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73b0b2 0x744660 0x5f13ea 0x73c308 0x73f5cf 0x48dcea 0x73f89c 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73b0b2 github.com/moonfrog/heimdall/service.(*Service).Send+0x442 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:79
# 0x744660 github.com/moonfrog/heimdall/service/manager.(*Sticky).Send+0x5e0 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:91
# 0x5f13ea github.com/moonfrog/heimdall/serviceController.(*Controller).Send+0x22a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:169
# 0x73c308 github.com/moonfrog/heimdall/service.(*Service).process+0x8a8 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:142
# 0x73f5cf github.com/moonfrog/heimdall/service.(*Service).readPump.func1.1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:111
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x73f89c github.com/moonfrog/heimdall/service.(*Service).readPump.func1+0x2bc /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:112
102 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x7457a6 0x5f23d9 0x5f9d26 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x7457a6 github.com/moonfrog/heimdall/service/manager.(*Sticky).Connect+0x856 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:147
# 0x5f23d9 github.com/moonfrog/heimdall/serviceController.(*Controller).OnConnect+0x3b9 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:235
# 0x5f9d26 github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0xe86 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:284
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
13 @ 0x42fd33 0x42fdf4 0x43f877 0x43f506 0x6299cc 0x62b7d9 0x62ccdc 0x640964 0x5e6306 0x73b75b 0x73cb72 0x74475e 0x5f1ea3 0x491dac 0x491c99 0x5fa486 0x5fa25a 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x43f506 net.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:52
# 0x6299cc net.(*fdMutex).RWLock+0x16c /usr/local/go/src/net/fd_mutex.go:143
# 0x62b7d9 net.(*netFD).writeLock+0x39 /usr/local/go/src/net/fd_unix.go:194
# 0x62ccdc net.(*netFD).Write+0x6c /usr/local/go/src/net/fd_unix.go:319
# 0x640964 net.(*conn).Write+0xe4 /usr/local/go/src/net/net.go:184
# 0x5e6306 github.com/moonfrog/heimdall/network.(*Messenger).WriteMessage+0x86 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:40
# 0x73b75b github.com/moonfrog/heimdall/service.(*Service).sendPacket+0x55b /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:91
# 0x73cb72 github.com/moonfrog/heimdall/service.(*Service).OnDisconnect+0xb2 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:179
# 0x74475e github.com/moonfrog/heimdall/service/manager.(*Sticky).Disconnect+0x5e /home/rts/gopath/src/github.com/moonfrog/heimdall/service/manager/sticky.go:97
# 0x5f1ea3 github.com/moonfrog/heimdall/serviceController.(*Controller).OnDisconnect+0x393 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:210
# 0x491dac github.com/moonfrog/heimdall/router.(*Router).onDisconnect+0x8c /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:151
# 0x491c99 github.com/moonfrog/heimdall/router.(*Router).UnregisterClient+0x309 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:146
# 0x5fa486 github.com/moonfrog/heimdall/client.(*Client).disconnect+0x206 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:329
# 0x5fa25a github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x13ba /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:274
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137
7 @ 0x42fd33 0x42a7ee 0x429cb0 0x629fba 0x62a026 0x62be7a 0x640644 0x611d09 0x612b7a 0x612eb9 0x5e60f4 0x73f614 0x460ae1
# 0x429cb0 net.runtime_pollWait+0x60 /usr/local/go/src/runtime/netpoll.go:160
# 0x629fba net.(*pollDesc).Wait+0x3a /usr/local/go/src/net/fd_poll_runtime.go:73
# 0x62a026 net.(*pollDesc).WaitRead+0x36 /usr/local/go/src/net/fd_poll_runtime.go:78
# 0x62be7a net.(*netFD).Read+0x23a /usr/local/go/src/net/fd_unix.go:250
# 0x640644 net.(*conn).Read+0xe4 /usr/local/go/src/net/net.go:172
# 0x611d09 bufio.(*Reader).fill+0x1e9 /usr/local/go/src/bufio/bufio.go:97
# 0x612b7a bufio.(*Reader).ReadSlice+0x21a /usr/local/go/src/bufio/bufio.go:328
# 0x612eb9 bufio.(*Reader).ReadBytes+0xa9 /usr/local/go/src/bufio/bufio.go:406
# 0x5e60f4 github.com/moonfrog/heimdall/network.(*Messenger).ReadMessage+0x54 /home/rts/gopath/src/github.com/moonfrog/heimdall/network/messenger.go:29
# 0x73f614 github.com/moonfrog/heimdall/service.(*Service).readPump.func1+0x34 /home/rts/gopath/src/github.com/moonfrog/heimdall/service/service.go:102
1 @ 0x42fd33 0x42fdf4 0x43f877 0x43f4c6 0x4c5704 0x4c6740 0x5f2d6a 0x5f5fcc 0x5f5dc4 0x5890f3 0x460ae1
# 0x43f4c6 sync.runtime_Semacquire+0x26 /usr/local/go/src/runtime/sema.go:47
# 0x4c5704 sync.(*Mutex).Lock+0x1c4 /usr/local/go/src/sync/mutex.go:83
# 0x4c6740 sync.(*RWMutex).Lock+0x30 /usr/local/go/src/sync/rwmutex.go:83
# 0x5f2d6a github.com/moonfrog/heimdall/serviceController.(*Controller).refreshServices+0x4a /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:257
# 0x5f5fcc github.com/moonfrog/heimdall/serviceController.(*Controller).init.func1+0x1ec /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/controller.go:333
# 0x5f5dc4 github.com/moonfrog/heimdall/serviceController.loadConfig.func2+0x84 /home/rts/gopath/src/github.com/moonfrog/heimdall/serviceController/configServices.go:84
# 0x5890f3 github.com/moonfrog/badger/zootils.(*ZooKeeper).watchFunc+0x53 /home/rts/gopath/src/github.com/moonfrog/badger/zootils/zootils.go:62
1 @ 0x42fd33 0x43e9f7 0x43df52 0x5fc579 0x491911 0x5faae9 0x5f94bf 0x5f8cdb 0x5ffd9f 0x48dcea 0x5f729f 0x460ae1
# 0x5fc579 github.com/moonfrog/heimdall/client.(*Client).RejectConnection+0x369 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:495
# 0x491911 github.com/moonfrog/heimdall/router.(*Router).RegisterClient+0x8a1 /home/rts/gopath/src/github.com/moonfrog/heimdall/router/router.go:109
# 0x5faae9 github.com/moonfrog/heimdall/client.(*Client).join+0x5e9 /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:379
# 0x5f94bf github.com/moonfrog/heimdall/client.(*Client).handleRouterMsg+0x61f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:265
# 0x5f8cdb github.com/moonfrog/heimdall/client.(*Client).process+0xedb /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:254
# 0x5ffd9f github.com/moonfrog/heimdall/client.(*Client).stateMachine.func1+0x3f /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:136
# 0x48dcea github.com/moonfrog/heimdall/common.Safe+0x3a /home/rts/gopath/src/github.com/moonfrog/heimdall/common/utils.go:49
# 0x5f729f github.com/moonfrog/heimdall/client.(*Client).stateMachine+0x1df /home/rts/gopath/src/github.com/moonfrog/heimdall/client/client.go:137