From 442887eb763203318da7194503976c346c2b0468 Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Fri, 15 Nov 2024 15:00:57 -0300 Subject: [PATCH 1/6] chore: report go runtime stats --- .gitignore | 1 + feedback/listener.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.gitignore b/.gitignore index e55e6bb..06f790b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ vendor/ *.swo .*swp +.vscode .idea .tool-versions diff --git a/feedback/listener.go b/feedback/listener.go index e05c636..758bc87 100644 --- a/feedback/listener.go +++ b/feedback/listener.go @@ -27,6 +27,7 @@ import ( "fmt" "os" "os/signal" + "runtime" "sync" "syscall" "time" @@ -132,6 +133,8 @@ func (l *Listener) Start() { l.Broker.Start() l.InvalidTokenHandler.Start() + go l.reportGoStats() + statsReporterReportMetricCount(l.StatsReporters, "feedback_listener_restart", 1, "", "") @@ -172,6 +175,23 @@ func (l *Listener) flushStats() { "invalid_token_handler_buffer", bufferSize, "", "") } +func (l *Listener) reportGoStats() { + for { + num := runtime.NumGoroutine() + m := &runtime.MemStats{} + runtime.ReadMemStats(m) + gcTime := m.PauseNs[(m.NumGC+255)%256] + for _, statsReporter := range l.StatsReporters { + statsReporter.ReportGoStats( + num, + m.Alloc, m.HeapObjects, m.NextGC, + gcTime, + ) + } + time.Sleep(30 * time.Second) + } +} + // Cleanup ends the Listener execution func (l *Listener) Cleanup() { l.flushStats() From aa61d21cb22bfcdddf8753ceb51ff73e3ac1a93d Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Fri, 15 Nov 2024 15:02:27 -0300 Subject: [PATCH 2/6] fix: misuse of unbuffered os.Signal channel as argument to signal.Notify --- feedback/listener.go | 2 +- pusher/pusher.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/feedback/listener.go b/feedback/listener.go index 758bc87..20361d6 100644 --- a/feedback/listener.go +++ b/feedback/listener.go @@ -138,7 +138,7 @@ func (l *Listener) Start() { statsReporterReportMetricCount(l.StatsReporters, "feedback_listener_restart", 1, "", "") - sigchan := make(chan os.Signal) + sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) flushTicker := time.NewTicker(l.statsFlushTime) diff --git a/pusher/pusher.go b/pusher/pusher.go index 7ae46fa..89752f2 100644 --- a/pusher/pusher.go +++ b/pusher/pusher.go @@ -118,7 +118,7 @@ func (p *Pusher) Start(ctx context.Context) { go p.Queue.ConsumeLoop(ctx) go p.reportGoStats() - sigchan := make(chan os.Signal) + sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) select { From f4f511fa2a6b9f3e0ffd31979f513de5c6a72fb0 Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Mon, 18 Nov 2024 10:31:51 -0300 Subject: [PATCH 3/6] chore: start pprof on loopback interface --- feedback/listener.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/feedback/listener.go b/feedback/listener.go index 20361d6..9921919 100644 --- a/feedback/listener.go +++ b/feedback/listener.go @@ -25,6 +25,8 @@ package feedback import ( "context" "fmt" + "net/http" + "net/http/pprof" "os" "os/signal" "runtime" @@ -129,6 +131,8 @@ func (l *Listener) Start() { ) log.Info("starting the feedback listener...") + go l.StartPprofServer() + go l.Queue.ConsumeLoop(context.Background()) l.Broker.Start() l.InvalidTokenHandler.Start() @@ -237,3 +241,20 @@ func WaitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { return true // timed out } } + +func (l *Listener) StartPprofServer() error { + mux := http.NewServeMux() + + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + server := &http.Server{ + Addr: "127.0.0.1:8081", + Handler: mux, + } + + return server.ListenAndServe() +} From 675b16d798fa681cbff7785b4603746be7589fd5 Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Mon, 18 Nov 2024 13:54:00 -0300 Subject: [PATCH 4/6] chore: parameterize host and port for pprof http server using env vars --- feedback/listener.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/feedback/listener.go b/feedback/listener.go index 9921919..ca70d57 100644 --- a/feedback/listener.go +++ b/feedback/listener.go @@ -78,6 +78,9 @@ func NewListener( func (l *Listener) loadConfigurationDefaults() { l.Config.SetDefault("feedbackListeners.gracefulShutdownTimeout", 1) l.Config.SetDefault("stats.flush.s", 5) + + l.Config.SetDefault("pprof.host", "127.0.0.1") + l.Config.SetDefault("pprof.port", 8081) } func (l *Listener) configure(statsdClientrOrNil interfaces.StatsDClient) error { @@ -251,8 +254,9 @@ func (l *Listener) StartPprofServer() error { mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + addr := fmt.Sprintf("%s:%d", l.Config.GetString("pprof.host"), l.Config.GetInt("pprof.port")) server := &http.Server{ - Addr: "127.0.0.1:8081", + Addr: addr, Handler: mux, } From 729cd74cec6e62f52a9352c5878eaeffb4c3e3ec Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Wed, 4 Dec 2024 17:06:07 -0300 Subject: [PATCH 5/6] chore: add pprof env vars to config yamls --- config/default.yaml | 3 +++ config/docker_test.yaml | 3 +++ config/test.yaml | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/default.yaml b/config/default.yaml index d9eb1b3..c22c2c0 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -55,6 +55,9 @@ stats: host: "localhost:8125" prefix: "push" buflen: 1 +pprof: + host: "127.0.0.1" + port: 8081 invalidToken: pg: table: "test_apns" diff --git a/config/docker_test.yaml b/config/docker_test.yaml index fe22f2f..e78534a 100644 --- a/config/docker_test.yaml +++ b/config/docker_test.yaml @@ -54,6 +54,9 @@ stats: host: "statsd:8125" prefix: "push" flushIntervalMs: 5000 +pprof: + host: "127.0.0.1" + port: 8081 invalidToken: handlers: - pg diff --git a/config/test.yaml b/config/test.yaml index 3086589..7c1f56a 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -51,7 +51,10 @@ stats: statsd: host: "localhost:40001" prefix: "push" - flushIntervalMs: 5000 + flushIntervalMs: +pprof: + host: "127.0.0.1" + port: 8081 invalidToken: handlers: - pg From 46f5a36a41da7cad91ab8eea9eae86ba2aeff7a4 Mon Sep 17 00:00:00 2001 From: "fernando.gava" Date: Fri, 6 Dec 2024 12:00:50 -0300 Subject: [PATCH 6/6] chore: add config to enable/disable pprof --- config/default.yaml | 1 + config/docker_test.yaml | 1 + config/test.yaml | 1 + feedback/listener.go | 5 ++++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/default.yaml b/config/default.yaml index c22c2c0..37ec2b0 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -56,6 +56,7 @@ stats: prefix: "push" buflen: 1 pprof: + enabled: false host: "127.0.0.1" port: 8081 invalidToken: diff --git a/config/docker_test.yaml b/config/docker_test.yaml index e78534a..46fbd3e 100644 --- a/config/docker_test.yaml +++ b/config/docker_test.yaml @@ -55,6 +55,7 @@ stats: prefix: "push" flushIntervalMs: 5000 pprof: + enabled: false host: "127.0.0.1" port: 8081 invalidToken: diff --git a/config/test.yaml b/config/test.yaml index 7c1f56a..8f5ad41 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -53,6 +53,7 @@ stats: prefix: "push" flushIntervalMs: pprof: + enabled: false host: "127.0.0.1" port: 8081 invalidToken: diff --git a/feedback/listener.go b/feedback/listener.go index ca70d57..cc895d4 100644 --- a/feedback/listener.go +++ b/feedback/listener.go @@ -79,6 +79,7 @@ func (l *Listener) loadConfigurationDefaults() { l.Config.SetDefault("feedbackListeners.gracefulShutdownTimeout", 1) l.Config.SetDefault("stats.flush.s", 5) + l.Config.SetDefault("pprof.enabled", false) l.Config.SetDefault("pprof.host", "127.0.0.1") l.Config.SetDefault("pprof.port", 8081) } @@ -134,7 +135,9 @@ func (l *Listener) Start() { ) log.Info("starting the feedback listener...") - go l.StartPprofServer() + if l.Config.GetBool("pprof.enabled") { + go l.StartPprofServer() + } go l.Queue.ConsumeLoop(context.Background()) l.Broker.Start()