Groundwork for OTel instrumentation in operator (#2272)

* Basic OTel infrastructure for metrics and traces to ingest directly into DT cluster, sample metric and spans.

* Move OTel basic setup stuff to utils. Add simple auto instrumentation.

* Use Noop implementation of OTel if no OTel config secret is provided.

* Start OTel collection with auto-instrumentation also in operator and CSI driver.

* Introduce a few OTel helpers that make calls resilient when OTel can't be properly started.

* fixup! Basic OTel infrastructure for metrics and traces to ingest directly into DT cluster, sample metric and spans.

* Fix typo

Co-authored-by: Albian Krasniqi <[email protected]>

* Add noop tracer installment log.

Co-authored-by: Albian Krasniqi <[email protected]>

* fixup! Basic OTel infrastructure for metrics and traces to ingest directly into DT cluster, sample metric and spans.

---------

Co-authored-by: Albian Krasniqi <[email protected]>

cmd/csi/provisioner/builder.go

@@ -1,11 +1,14 @@
 package provisioner
 
 import (
+	"context"
+
 	"github.com/Dynatrace/dynatrace-operator/cmd/config"
 	cmdManager "github.com/Dynatrace/dynatrace-operator/cmd/manager"
 	dtcsi "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi"
 	"github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi/metadata"
 	csiprovisioner "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi/provisioner"
+	"github.com/Dynatrace/dynatrace-operator/pkg/util/otel"
 	"github.com/Dynatrace/dynatrace-operator/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
@@ -111,6 +114,9 @@ func (builder CommandBuilder) buildRun() func(*cobra.Command, []string) error {
 			return err
 		}
 
+		otelShutdownFn := otel.Start(context.Background(), "dynatrace-csi-provisioner", csiManager.GetAPIReader(), builder.namespace)
+		defer otelShutdownFn()
+
 		err = createCsiDataPath(builder.getFilesystem())
 		if err != nil {
 			return err

cmd/csi/provisioner/command_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/Dynatrace/dynatrace-operator/cmd/config"
 	cmdManager "github.com/Dynatrace/dynatrace-operator/cmd/manager"
+	dtfake "github.com/Dynatrace/dynatrace-operator/pkg/api/scheme/fake"
 	dtcsi "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/logger"
 	"github.com/spf13/afero"
@@ -17,7 +18,9 @@ func TestCsiCommand(t *testing.T) {
 	configProvider := &config.MockProvider{}
 	configProvider.On("GetConfig").Return(&rest.Config{}, nil)
 
+	clt := dtfake.NewClient()
 	cmdMgr := &cmdManager.MockManager{}
+	cmdMgr.On("GetAPIReader", mock.Anything, mock.Anything).Return(clt, nil)
 
 	managerProvider := &cmdManager.MockProvider{}
 	managerProvider.On("CreateManager", mock.Anything, mock.Anything).Return(cmdMgr, nil)

cmd/csi/server/builder.go

@@ -1,11 +1,14 @@
 package server
 
 import (
+	"context"
+
 	"github.com/Dynatrace/dynatrace-operator/cmd/config"
 	cmdManager "github.com/Dynatrace/dynatrace-operator/cmd/manager"
 	dtcsi "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi"
 	csidriver "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi/driver"
 	"github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi/metadata"
+	"github.com/Dynatrace/dynatrace-operator/pkg/util/otel"
 	"github.com/Dynatrace/dynatrace-operator/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
@@ -115,6 +118,9 @@ func (builder CommandBuilder) buildRun() func(*cobra.Command, []string) error {
 			return err
 		}
 
+		otelShutdownFn := otel.Start(context.Background(), "dynatrace-csi-server", csiManager.GetAPIReader(), builder.namespace)
+		defer otelShutdownFn()
+
 		err = createCsiDataPath(builder.getFilesystem())
 		if err != nil {
 			return err

cmd/csi/server/command_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/Dynatrace/dynatrace-operator/cmd/config"
 	cmdManager "github.com/Dynatrace/dynatrace-operator/cmd/manager"
+	dtfake "github.com/Dynatrace/dynatrace-operator/pkg/api/scheme/fake"
 	dtcsi "github.com/Dynatrace/dynatrace-operator/pkg/controllers/csi"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/logger"
 	"github.com/spf13/afero"
@@ -17,7 +18,9 @@ func TestCsiCommand(t *testing.T) {
 	configProvider := &config.MockProvider{}
 	configProvider.On("GetConfig").Return(&rest.Config{}, nil)
 
+	clt := dtfake.NewClient()
 	cmdMgr := &cmdManager.MockManager{}
+	cmdMgr.On("GetAPIReader", mock.Anything, mock.Anything).Return(clt, nil)
 
 	managerProvider := &cmdManager.MockProvider{}
 	managerProvider.On("CreateManager", mock.Anything, mock.Anything).Return(cmdMgr, nil)

cmd/manager/test.go

@@ -23,7 +23,7 @@ func (mgr *TestManager) GetClient() client.Client {
 }
 
 func (mgr *TestManager) GetAPIReader() client.Reader {
-	return struct{ client.Reader }{}
+	return nil
 }
 
 func (mgr *TestManager) GetControllerOptions() config.Controller {

cmd/manager/test_test.go

@@ -12,7 +12,6 @@ func TestTestManager(t *testing.T) {
 	mgr := TestManager{}
 
 	assert.NotNil(t, mgr.GetClient())
-	assert.NotNil(t, mgr.GetAPIReader())
 	assert.NotNil(t, mgr.GetControllerOptions())
 	assert.Equal(t, scheme.Scheme, mgr.GetScheme())
 	assert.NotNil(t, mgr.GetLogger())

cmd/operator/builder.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Dynatrace/dynatrace-operator/pkg/controllers/certificates"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/kubeobjects"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/kubesystem"
+	"github.com/Dynatrace/dynatrace-operator/pkg/util/otel"
 	"github.com/Dynatrace/dynatrace-operator/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -172,11 +173,13 @@ func (builder CommandBuilder) runBootstrapper(kubeCfg *rest.Config) error {
 
 func (builder CommandBuilder) runOperatorManager(kubeCfg *rest.Config, isDeployedViaOlm bool) error {
 	operatorManager, err := builder.getOperatorManagerProvider(isDeployedViaOlm).CreateManager(builder.namespace, kubeCfg)
-
 	if err != nil {
 		return err
 	}
 
+	otelShutdownFn := otel.Start(context.Background(), "dynatrace-operator", operatorManager.GetAPIReader(), builder.namespace)
+	defer otelShutdownFn()
+
 	err = operatorManager.Start(builder.getSignalHandler())
 
 	return errors.WithStack(err)

cmd/operator/builder_test.go

@@ -195,8 +195,10 @@ func TestOperatorCommand(t *testing.T) {
 			On("CreateManager", mock.AnythingOfType("string"), &rest.Config{}).
 			Return(bootstrapMockMgr, nil)
 
+		clt := dtfake.NewClient()
 		operatorMockMgr := &manager.MockManager{}
 		operatorMockMgr.On("Start", mock.Anything).Return(nil)
+		operatorMockMgr.On("GetAPIReader", mock.Anything).Return(clt, nil)
 
 		mockOperatorMgrProvider := &manager.MockProvider{}
 		mockOperatorMgrProvider.

cmd/webhook/builder.go

@@ -10,6 +10,7 @@ import (
 	dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/pkg/api/v1beta1/dynakube"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/kubeobjects"
 	"github.com/Dynatrace/dynatrace-operator/pkg/util/kubesystem"
+	"github.com/Dynatrace/dynatrace-operator/pkg/util/otel"
 	"github.com/Dynatrace/dynatrace-operator/pkg/version"
 	"github.com/Dynatrace/dynatrace-operator/pkg/webhook"
 	"github.com/Dynatrace/dynatrace-operator/pkg/webhook/mutation/namespace_mutator"
@@ -122,6 +123,9 @@ func (builder CommandBuilder) buildRun() func(*cobra.Command, []string) error {
 			return err
 		}
 
+		otelShutdownFn := otel.Start(context.Background(), "dynatrace-webhook", webhookManager.GetAPIReader(), builder.namespace)
+		defer otelShutdownFn()
+
 		err = startCertificateWatcher(webhookManager, builder.namespace, builder.podName)
 		if err != nil {
 			return err

doc/otel.md

@@ -0,0 +1,30 @@
+# How to setup OpenTelementry
+
+Dynatrace operator, CSI driver and webhook are instrumented using OpenTelemetry. To enable this instrumentation and ingest collected
+metrics and traces into your tenant follow this guide.
+
+## Create an access token with the following scopes
+
+- openTelemetryTrace.ingest
+- metrics.ingest
+- logs.ingest
+
+## Create OpenTelementry configuration secret
+
+```yaml
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: dynatrace-operator-otel-config
+  namespace: dynatrace
+data:
+  endpoint: base64(<uuid>.dev.dyntracelabs.com)
+  apiToken: base64(<apiToken>)
+```
+
+*Note:*
+
+- as indicated the values have to be base64 encoded (as usually with K8S)
+- obey to the name
+- make sure it is created in the same namespace as the webhook

go.mod

@@ -17,6 +17,16 @@ require (
 	github.com/spf13/afero v1.10.0
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/contrib/instrumentation/host v0.45.0
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0
+	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
+	go.opentelemetry.io/otel/metric v1.19.0
+	go.opentelemetry.io/otel/sdk v1.19.0
+	go.opentelemetry.io/otel/sdk/metric v1.19.0
+	go.opentelemetry.io/otel/trace v1.19.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/net v0.17.0
@@ -37,6 +47,7 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -46,7 +57,9 @@ require (
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
@@ -57,10 +70,12 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -70,13 +85,21 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.8 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/vladimirvivien/gexe v0.2.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/term v0.13.0 // indirect