Skip to content
This repository has been archived by the owner on Jul 4, 2024. It is now read-only.

Commit

Permalink
Grafana dashboard for system broker (#1839)
Browse files Browse the repository at this point in the history
* introduce metrics collector

* create path to instrumentation mapping

* instrument handlers

* add policy and service-monitor

* fix path to instrumentation mapping

* add metrics config

* add custom buckets to catalog endpoint request duration histogram

* fix instrumentation

* add metrics server

* register metrics collector

* add dashboard configmap

* make imports

* adjust component PR reference

* handle error

* make imports

* fix typo
  • Loading branch information
StanislavStefanov authored Apr 22, 2021
1 parent fdff31b commit efa436a
Show file tree
Hide file tree
Showing 15 changed files with 2,157 additions and 76 deletions.
1,729 changes: 1,729 additions & 0 deletions chart/compass/charts/system-broker/templates/dashboard-configmap.yaml

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions chart/compass/charts/system-broker/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ spec:
value: "https://{{ .Values.global.gateway.mtls.host }}.{{ .Values.global.ingress.domainName }}"
- name: ORD_STATIC_PATH
value: "{{ .Values.global.ordService.staticPrefix }}"
- name: METRICS_PORT
value: "{{ .Values.metrics.port }}"
livenessProbe:
httpGet:
path: /healthz
Expand Down
12 changes: 12 additions & 0 deletions chart/compass/charts/system-broker/templates/policy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Required because Prometheus Operator doesn't have Istio Sidecar
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
name: {{ template "fullname" . }}
spec:
selector:
matchLabels:
app: {{ .Chart.Name }}
portLevelMtls:
{{ .Values.metrics.port }}:
mode: "PERMISSIVE"
26 changes: 26 additions & 0 deletions chart/compass/charts/system-broker/templates/service-monitor.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{{- if eq .Values.global.metrics.enabled true -}}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ template "fullname" . }}
labels:
prometheus: monitoring
app: {{ .Chart.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
release: "{{ .Release.Name }}"
heritage: "{{ .Release.Service }}"
spec:
endpoints:
- port: metrics
path: /metrics
metricRelabelings:
- sourceLabels: [ __name__ ]
regex: ^(go_gc_duration_seconds|go_goroutines|go_memstats_alloc_bytes|go_memstats_heap_alloc_bytes|go_memstats_heap_inuse_bytes|go_memstats_heap_sys_bytes|go_memstats_stack_inuse_bytes|go_threads|http_requests_total|process_cpu_seconds_total|process_max_fds|process_open_fds|process_resident_memory_bytes|process_start_time_seconds|process_virtual_memory_bytes|go_sql_stats_connections_in_use|go_sql_stats_connections_idle|go_sql_stats_connections_max_open|compass_system_broker_catalog_request_duration_seconds_bucket|compass_system_broker_catalog_request_duration_seconds_count|compass_system_broker_catalog_request_duration_seconds_sum|compass_system_broker_catalog_response_size_bucket|compass_system_broker_catalog_response_size_count|compass_system_broker_catalog_response_size_sum|compass_system_broker_catalog_request_total|compass_system_broker_provision_request_duration_seconds_bucket|compass_system_broker_provision_request_duration_seconds_count|compass_system_broker_provision_request_duration_seconds_sum|compass_system_broker_provision_response_size_bucket|compass_system_broker_provision_response_size_count|compass_system_broker_provision_response_size_sum|compass_system_broker_provision_request_total|compass_system_broker_deprovision_request_duration_seconds_bucket|compass_system_broker_deprovision_request_duration_seconds_count|compass_system_broker_deprovision_request_duration_seconds_sum|compass_system_broker_deprovision_response_size_bucket|compass_system_broker_deprovision_response_size_count|compass_system_broker_deprovision_response_size_sum|compass_system_broker_deprovision_request_total|compass_system_broker_bind_request_duration_seconds_bucket|compass_system_broker_bind_request_duration_seconds_count|compass_system_broker_bind_request_duration_seconds_sum|compass_system_broker_bind_response_size_bucket|compass_system_broker_bind_response_size_count|compass_system_broker_bind_response_size_sum|compass_system_broker_bind_request_total|compass_system_broker_unbind_request_duration_seconds_bucket|compass_system_broker_unbind_request_duration_seconds_count|compass_system_broker_unbind_request_duration_seconds_sum|compass_system_broker_unbind_response_size_bucket|compass_system_broker_unbind_response_size_count|compass_system_broker_unbind_response_size_sum|compass_system_broker_unbind_request_total)$
action: keep
namespaceSelector:
matchNames:
- "{{ .Release.Namespace }}"
selector:
matchLabels:
app: {{ .Chart.Name }}
{{- end }}
2 changes: 1 addition & 1 deletion chart/compass/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ global:
version: "PR-1832"
system_broker:
dir:
version: "PR-1828"
version: "PR-1839"
certs_setup_job:
containerRegistry:
path: eu.gcr.io/kyma-project
Expand Down
26 changes: 24 additions & 2 deletions components/system-broker/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ package main
import (
"context"
"os"
"sync"

"github.com/kyma-incubator/compass/components/system-broker/internal/metrics"
"github.com/kyma-incubator/compass/components/system-broker/pkg/oauth"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/gorilla/mux"

Expand Down Expand Up @@ -52,21 +56,39 @@ func main() {
err = cfg.Validate()
fatalOnError(err)

metrics.MapPathToInstrumentation(cfg.Server.RootAPI)

ctx, err = log.Configure(ctx, cfg.Log)
fatalOnError(err)

directorGraphQLClient, err := graphql.PrepareGqlClient(cfg.GraphQLClient, cfg.HttpClient, oauth.NewTokenAuthorizationProviderFromHeader())
fatalOnError(err)

systemBroker := osb.NewSystemBroker(directorGraphQLClient, cfg.ORD.ServiceURL+cfg.ORD.StaticPath)
osbApi := osb.API(cfg.Server.RootAPI, systemBroker, sblog.NewDefaultLagerAdapter())
collector := metrics.NewCollector()
osbApi := osb.API(cfg.Server.RootAPI, systemBroker, sblog.NewDefaultLagerAdapter(), collector)
prometheus.MustRegister(collector)

middlewares := []mux.MiddlewareFunc{
httputil.HeaderForwarder(cfg.HttpClient.ForwardHeaders),
}
srv := server.New(cfg.Server, middlewares, osbApi)

srv.Start(ctx)
metricsHandler := mux.NewRouter()
metricsHandler.Handle("/metrics", promhttp.Handler())
metricsServer := server.NewServerWithRouter(&server.Config{
Port: cfg.Metrics.Port,
Timeout: cfg.Metrics.Timeout,
ShutdownTimeout: cfg.Metrics.ShutDownTimeout,
}, metricsHandler)

wg := &sync.WaitGroup{}
wg.Add(2)

go metricsServer.Start(ctx, wg)
go srv.Start(ctx, wg)

wg.Wait()
}

func fatalOnError(err error) {
Expand Down
18 changes: 2 additions & 16 deletions components/system-broker/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
Expand Down Expand Up @@ -510,8 +511,6 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.9 h1:5OCMOdde1TCT2sookEuVeEZzA8bmRSFV3AwPDZAG8AA=
github.com/klauspost/compress v1.11.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
Expand All @@ -532,12 +531,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kyma-incubator/compass/components/connector v0.0.0-20210315172259-e186b4cac80b/go.mod h1:bPwjvhAV9q4N70h30Ow745LRQftp0JhnLa7bJK5cEKs=
github.com/kyma-incubator/compass/components/director v0.0.0-20210222123648-c454983cf74b/go.mod h1:OB4+ZBUsOWPL8Y6FXIjyaTHFGUHW1UUFVWwNGz0pLRM=
github.com/kyma-incubator/compass/components/director v0.0.0-20210301143156-3b39f117e7bf/go.mod h1:hJ3bgWfwFhdnxZF7Gcok5Rm1zDT4bHtVGHDeXDBvTfg=
github.com/kyma-incubator/compass/components/director v0.0.0-20210330105728-6ca6d1a07081 h1:B5YW5qCt1PeNkNq1eHgS8O9+Ac6M05CJTn7hLVOzTWg=
github.com/kyma-incubator/compass/components/director v0.0.0-20210330105728-6ca6d1a07081/go.mod h1:4u8ypBtJh66WYOouCuKpMJ70XCK29F4JoyKD/Mp6o1M=
github.com/kyma-incubator/compass/components/director v0.0.0-20210412125407-088a6bd2781d h1:H+SkznDET0oBiIGkvmiafiSsvDlCcWQaFIud9GmYZP4=
github.com/kyma-incubator/compass/components/director v0.0.0-20210412125407-088a6bd2781d/go.mod h1:dBSz67XgayIOfFrbIVn7aB1NJM/waXh0Yb+4h7omaww=
github.com/kyma-incubator/compass/components/director v0.0.0-20210415152921-17feb4dbc6b1 h1:YrfAtj5X3n1+ZScYt79eF/HhBTDOTlkoEYLIRK3wkhI=
github.com/kyma-incubator/compass/components/director v0.0.0-20210415152921-17feb4dbc6b1/go.mod h1:dBSz67XgayIOfFrbIVn7aB1NJM/waXh0Yb+4h7omaww=
github.com/kyma-incubator/compass/components/director v0.0.0-20210416142045-25b90bbc9ee6 h1:WyTOnAQS73BFubJQujhGMQtHvIOervZPJKBpQJLf1sY=
github.com/kyma-incubator/compass/components/director v0.0.0-20210416142045-25b90bbc9ee6/go.mod h1:dBSz67XgayIOfFrbIVn7aB1NJM/waXh0Yb+4h7omaww=
github.com/kyma-incubator/compass/components/operations-controller v0.0.0-20210213091620-beb5492e9d8b/go.mod h1:201wjhJxyaSA4pVp33VKOp91oNC+PQR591haj1t7KhA=
Expand Down Expand Up @@ -595,8 +588,6 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.3/go.mod h1:1ftk08SazyElaaNvmqAfZWGwJzshjCfBXDLoQtPAMNk=
github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0 h1:8E6DrFvII6QR4eJ3PkFvV+lc03P+2qwqTPLm1ax7694=
github.com/maxbrunsfeld/counterfeiter/v6 v6.3.0/go.mod h1:fcEyUyXZXoV4Abw8DX0t7wyL8mCDxXyU4iAFZfT3IHw=
github.com/maxbrunsfeld/counterfeiter/v6 v6.4.1 h1:hZD/8vBuw7x1WqRXD/WGjVjipbbo/HcDBgySYYbrUSk=
github.com/maxbrunsfeld/counterfeiter/v6 v6.4.1/go.mod h1:DK1Cjkc0E49ShgRVs5jy5ASrM15svSnem3K/hiSGD8o=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
Expand Down Expand Up @@ -666,9 +657,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U=
github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ=
github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
Expand Down Expand Up @@ -854,8 +845,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasthttp v1.19.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/fasthttp v1.22.0 h1:OpwH5KDOJ9cS2bq8fD+KfT4IrksK0llvkHf4MZx42jQ=
github.com/valyala/fasthttp v1.22.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs=
github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
Expand Down Expand Up @@ -1018,10 +1007,8 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
Expand Down Expand Up @@ -1195,7 +1182,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 h1:K+NlvTLy0oONtRtkl1jRD9xIhnItbG2PiE7YOdjPb+k=
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
Expand Down
4 changes: 4 additions & 0 deletions components/system-broker/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package config
import (
"reflect"

"github.com/kyma-incubator/compass/components/system-broker/internal/metrics"

"github.com/kyma-incubator/compass/components/system-broker/pkg/ord"

"github.com/kyma-incubator/compass/components/director/pkg/log"
Expand All @@ -42,6 +44,7 @@ type Config struct {
GraphQLClient *graphql.Config `mapstructure:"graphql_client"`
OAuthProvider *oauth.Config `mapstructure:"oauth_provider"`
ORD *ord.Config `mapstructure:"ord"`
Metrics *metrics.Config `mapstructure:"metrics"`
}

func AddPFlags(set *pflag.FlagSet) {
Expand All @@ -57,6 +60,7 @@ func DefaultConfig() *Config {
GraphQLClient: graphql.DefaultConfig(),
OAuthProvider: oauth.DefaultConfig(),
ORD: ord.DefaultConfig(),
Metrics: metrics.DefaultConfig(),
}
}

Expand Down
Loading

0 comments on commit efa436a

Please sign in to comment.