feat: e2e tests with helm and k3d

.github/workflows/master.yaml

@@ -21,21 +21,30 @@ jobs:
       - name: helm lint
         run: |
           helm lint . --strict
-          helm lint . --strict -f test-linter/values-lint.yaml
-          helm dep up test-linter/subchart && helm lint test-linter/subchart --strict
+          helm lint . --strict -f test/linter/values-lint.yaml
+          helm dep up test/linter/subchart && helm lint test/linter/subchart --strict
       - uses: volesen/[email protected]
         with:
           version: 'v1.20.0'
-      - run: |
-          skaffold version
-      - uses: AbsaOSS/[email protected]
-        name: create cluster
+      - name: start k8s with k3d
+        uses: AbsaOSS/[email protected]
         with:
           cluster-name: "cmak"
           use-default-registry: true
           args: >-
             --config ./k3d.yaml
+      - name: build cmak-operator
+        run: |
+          skaffold config set default-repo "registry.localhost:5000"
+          skaffold build
+      - name: deploy cmak-operator
+        run: |
+          set -x
+          set -e
+          skaffold config set default-repo "registry.localhost:5000"
+          skaffold run -p e2e
       - name: e2e test
         run: |
-          sleep 20
-          ./e2e.sh
+          set -x
+          set -e
+          helm test cmak-operator --logs

Dockerfile

@@ -1,7 +1,9 @@
-FROM python:3.8-alpine3.11
+FROM python:3.8-alpine3.12
 
 COPY *.py /opt/
 
-RUN pip install click~=7.0 kazoo~=2.8
+RUN set -x && \
+  apk add --update --no-cache jq curl && \
+  pip install click~=7.0 kazoo~=2.8
 
 WORKDIR /opt

skaffold.yaml

@@ -12,6 +12,7 @@ profiles:
   activation:
   - command: run
   - command: dev
+  - command: delete
   deploy:
     helm:
       releases:
@@ -24,10 +25,16 @@ profiles:
           image: eshepelyuk/json2zk
         imageStrategy:
           helm: {}
-  portForward:
-  - resourceType: Service
-    resourceName: cmak
-    port: 2181
-  - resourceType: Service
-    resourceName: cmak
-    port: 9000
+- name: e2e
+  deploy:
+    helm:
+      releases:
+      - name: cmak-operator
+        chartPath: .
+        wait: true
+        valuesFiles:
+        - test/e2e/values-e2e.yaml
+        artifactOverrides:
+          image: eshepelyuk/json2zk
+        imageStrategy:
+          helm: {}

templates/cronjob.yaml → templates/cronjob-reconcile.yaml

@@ -4,7 +4,7 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: {{ $nm }}-clusters
+  name: {{ $nm }}-reconcile
   labels:
     {{- include "cmak.labels" . | nindent 4 }}
 spec:

templates/job-setup.yaml

@@ -0,0 +1,33 @@
+{{- $nm := include "cmak.name" . -}}
+{{- $dict := .Values.image | default (dict "tag" .Chart.Version "repository" (printf "%s/eshepelyuk/json2zk" .Values.imageRegistry)) -}}
+{{- $image := printf "%s:%s" $dict.repository $dict.tag -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ $nm }}-setup
+  labels:
+    {{- include "cmak.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: {{ $nm }}-clusters
+          configMap:
+            name: {{ $nm }}-clusters
+      containers:
+        - name: setup
+          image: {{ $image | quote }}
+          command: ['python3', './json2zk.py']
+          args:
+            - '{{ .Values.reconcile.overwriteZk | ternary "--" "--no-" }}overwrite-zk'
+            - '{{ $nm }}.{{ $.Release.Namespace }}:2181'
+            - '/kafka-manager/configs'
+            - '/opt/clusters'
+          volumeMounts:
+            - name: {{ $nm }}-clusters
+              mountPath: /opt/clusters

templates/tests/values-clusters.yaml

@@ -0,0 +1,23 @@
+{{- $nm := include "cmak.name" . -}}
+{{- $uiUrl := printf "http://%v.%v:%v/api/status/clusters" $nm .Release.Namespace .Values.ui.port -}}
+{{- $dict := .Values.image | default (dict "tag" .Chart.Version "repository" (printf "%s/eshepelyuk/json2zk" .Values.imageRegistry)) -}}
+{{- $image := printf "%s:%s" $dict.repository $dict.tag -}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ $nm }}-values-clusters
+  annotations:
+    "helm.sh/hook": test
+spec:
+  restartPolicy: Never
+  containers:
+  - name: values-clusters
+    image: {{ $image | quote }}
+    command:
+    - '/bin/sh'
+    - '-c'
+    - |
+      set -x
+      set -e
+      [ "$(curl -s {{ $uiUrl }} | jq '.clusters.active | length')" -eq 2 ]
+      curl -s {{ $uiUrl }} | jq '.clusters.active[].name'

test/e2e/values-e2e.yaml

@@ -0,0 +1,9 @@
+cmak:
+  clusters:
+  - name: cluster
+    curatorConfig:
+      zkConnect: kafka1.local
+  - name: cluster-disabled
+    enabled: false
+    curatorConfig:
+      zkConnect: kafka2.local

test-linter/subchart/Chart.yaml → test/linter/subchart/Chart.yaml

@@ -7,4 +7,4 @@ appVersion: 0.0.0
 dependencies:
 - name: cmak-operator
   version: 0.0.0
-  repository: file://../..
+  repository: file://../../..

test-linter/subchart/values.yaml → test/linter/subchart/values.yaml

@@ -6,3 +6,7 @@ propStr: "true"
 
 global:
   globalStr: "false"
+
+image:
+  tag: xxx
+  repository: yyy

values.schema.json

@@ -90,7 +90,7 @@
       "required": ["schedule"], "title": "reconciliation job config",
       "properties": {
         "schedule": {
-          "type": "string", "default": "*/2 * * * *", "title": "cron expression for periodic reconciliation"
+          "type": "string", "default": "*/3 * * * *", "title": "cron expression for periodic reconciliation"
         },
         "overwriteZk": {
           "type": "boolean", "default": true, "title": "allow overwrite Zookeeper settings of CMAK"

values.yaml

@@ -44,7 +44,7 @@ cmak:
   clusters: []
 
 reconcile:
-  schedule: "*/2 * * * *"
+  schedule: "*/3 * * * *"
   overwriteZk: true
   successfulJobsHistoryLimit: null
   failedJobsHistoryLimit: null