add detailed status, new CRD

angeloxx · Sep 1, 2024 · 00cdd07 · 00cdd07
1 parent 10e4b9f
commit 00cdd07
Show file tree

Hide file tree

Showing 7 changed files with 210 additions and 85 deletions.
diff --git a/api/v2/haegressgatewaypolicy_types.go b/api/v2/haegressgatewaypolicy_types.go
@@ -22,20 +22,23 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
 // HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy
 type HAEgressGatewayPolicyStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
 	ServiceCreated bool `json:"serviceCreated"`
 	PolicyCreated  bool `json:"policyCreated"`
+
+	// +kubebuilder:validation:Optional
+	ExitNode string `json:"exitNode,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	IPAddress string `json:"ipAddress,omitempty"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:resource:scope=Cluster
+//+kubebuilder:printcolumn:name="IP Address",type=string,JSONPath=`.status.ipAddress`
+//+kubebuilder:printcolumn:name="Exit Node",type=string,JSONPath=`.status.exitNode`
 
 // haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies API
 type HAEgressGatewayPolicy struct {

diff --git a/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml b/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml
@@ -12,11 +12,16 @@ spec:
     listKind: HAEgressGatewayPolicyList
     plural: haegressgatewaypolicies
     singular: haegressgatewaypolicy
-    shortNames:
-      - haegp
   scope: Cluster
   versions:
-    - name: v2
+    - additionalPrinterColumns:
+        - jsonPath: .status.ipAddress
+          name: IP Address
+          type: string
+        - jsonPath: .status.exitNode
+          name: Exit Node
+          type: string
+      name: v2
       schema:
         openAPIV3Schema:
           description: haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies
@@ -253,12 +258,13 @@ spec:
             status:
               description: HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy
               properties:
+                exitNode:
+                  type: string
+                ipAddress:
+                  type: string
                 policyCreated:
                   type: boolean
                 serviceCreated:
-                  description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
-                  of cluster Important: Run "make" to regenerate code after modifying
-                  this file'
                   type: boolean
               required:
                 - policyCreated

diff --git a/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml b/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml
@@ -12,11 +12,16 @@ spec:
     listKind: HAEgressGatewayPolicyList
     plural: haegressgatewaypolicies
     singular: haegressgatewaypolicy
-    shortNames:
-      - haegp
   scope: Cluster
   versions:
-  - name: v2
+  - additionalPrinterColumns:
+    - jsonPath: .status.ipAddress
+      name: IP Address
+      type: string
+    - jsonPath: .status.exitNode
+      name: Exit Node
+      type: string
+    name: v2
     schema:
       openAPIV3Schema:
         description: haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies
@@ -253,12 +258,13 @@ spec:
           status:
             description: HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy
             properties:
+              exitNode:
+                type: string
+              ipAddress:
+                type: string
               policyCreated:
                 type: boolean
               serviceCreated:
-                description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
-                  of cluster Important: Run "make" to regenerate code after modifying
-                  this file'
                 type: boolean
             required:
             - policyCreated

diff --git a/controllers/haegressgatewaypolicies_controller.go b/controllers/haegressgatewaypolicies_controller.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	haegressv2 "github.com/angeloxx/cilium-haegress-operator/api/v2"
 	haegressip "github.com/angeloxx/cilium-haegress-operator/pkg"
+	haegressiputil "github.com/angeloxx/cilium-haegress-operator/util"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -31,8 +32,13 @@ import (
 	"k8s.io/client-go/tools/record"
 	"reflect"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 // HAEgressGatewayPolicyReconciler reconciles a HAEgressGatewayPolicy object
@@ -74,23 +80,6 @@ func (r *HAEgressGatewayPolicyReconciler) Reconcile(ctx context.Context, req ctr
 		return ctrl.Result{}, err
 	}
 
-	/*
-		serviceNamespace := r.EgressNamespace
-		if haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] != "" {
-			serviceNamespace = haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace]
-		}
-		leaseExpectedName := fmt.Sprintf("cilium-l2announce-%s-%s",
-			serviceNamespace, haEgressGatewayPolicy.Name)
-		if haEgressGatewayPolicy.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] != leaseExpectedName {
-			haEgressGatewayPolicy.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] = leaseExpectedName
-
-			if err := r.Update(ctx, &haEgressGatewayPolicy); err != nil {
-				log.Error(err, "unable to update HAEgressGatewayPolicy, please check RBAC permissions")
-				return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, err
-			}
-		}
-
-	*/
 	if err := r.UpdateOrCreateCiliumEgressGatewayPolicy(ctx, &haEgressGatewayPolicy); err != nil {
 		log.Error(err, "unable to create or update CiliumEgressGatewayPolicy, please check RBAC permissions")
 		return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, err
@@ -113,11 +102,6 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateCiliumEgressGatewayPolic
 	if haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] != "" {
 		serviceNamespace = haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace]
 	}
-	/*
-		leaseExpectedName := fmt.Sprintf("cilium-l2announce-%s-%s",
-			serviceNamespace, haEgressGatewayPolicy.Name)
-		ciliumEgressGatewayPolicyNew.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] = leaseExpectedName
-	*/
 
 	ciliumEgressGatewayPolicyNew := &ciliumv2.CiliumEgressGatewayPolicy{
 		ObjectMeta: metav1.ObjectMeta{
@@ -154,6 +138,18 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateCiliumEgressGatewayPolic
 		if err := controllerutil.SetControllerReference(haEgressGatewayPolicy, ciliumEgressGatewayPolicyNew, r.Scheme); err != nil {
 			return err
 		}
+
+		// If service already exists, reconcile
+		service := &corev1.Service{}
+		err = r.Get(ctx, types.NamespacedName{Name: haEgressGatewayPolicy.Name, Namespace: serviceNamespace}, service)
+		if err == nil {
+			// Call the services reconcile function
+			_, syncError := haegressiputil.SyncServiceWithCiliumEgressGatewayPolicy(ctx, r.Client, logger, r.Recorder, *service, *ciliumEgressGatewayPolicyNew)
+			if syncError != nil {
+				return syncError
+			}
+		}
+
 	} else if err != nil {
 		return err
 	} else {
@@ -226,7 +222,7 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont
 		service.Annotations = make(map[string]string)
 	}
 	// Avoid L2 announcement by Cilium
-	service.Labels["service.kubernetes.io/service-proxy-name"] = "kubevip-managed-by-cilium-haegess"
+	service.Labels[haegressip.KubernetesServiceProxyNameAnnotation] = "kubevip-managed-by-cilium-haegess"
 	service.Labels[haegressip.HAEgressGatewayPolicyNamespace] = serviceNamespace
 	service.Labels[haegressip.HAEgressGatewayPolicyName] = haEgressGatewayPolicy.Name
 
@@ -241,8 +237,10 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont
 	if err != nil && apierrors.IsNotFound(err) {
 		log.Info("Creating a new Service for HAEgressGatewayPolicy", "Service.Namespace", service.Namespace, "Service.Name", service.Name)
 		err = r.Create(ctx, service)
-		r.Recorder.Event(haEgressGatewayPolicy, corev1.EventTypeNormal, "Created", "Service created")
-
+		r.Recorder.Event(haEgressGatewayPolicy,
+			corev1.EventTypeNormal,
+			"Created",
+			fmt.Sprintf("Service %s/%s created", service.Namespace, service.Name))
 		if err != nil {
 			return err
 		}
@@ -271,9 +269,63 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont
 	return nil
 }
 
+func (r *HAEgressGatewayPolicyReconciler) findObjectsForHaegressGatewayPolicy(ctx context.Context, obj client.Object) []reconcile.Request {
+	ownerRefs := obj.GetOwnerReferences()
+	requests := []reconcile.Request{}
+
+	for _, ownerRef := range ownerRefs {
+		if ownerRef.Kind == "HAEgressGatewayPolicy" {
+			requests = append(requests, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      ownerRef.Name,
+					Namespace: obj.GetNamespace(),
+				},
+			})
+		}
+	}
+
+	return requests
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *HAEgressGatewayPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&haegressv2.HAEgressGatewayPolicy{}).
+		Watches(
+			&corev1.Service{},
+			handler.EnqueueRequestsFromMapFunc(r.findObjectsForHaegressGatewayPolicy),
+			builder.WithPredicates(predicate.Funcs{
+				DeleteFunc: func(e event.DeleteEvent) bool {
+					return true
+				},
+				CreateFunc: func(e event.CreateEvent) bool {
+					return false
+				},
+				UpdateFunc: func(e event.UpdateEvent) bool {
+					return false
+				},
+				GenericFunc: func(e event.GenericEvent) bool {
+					return false
+				},
+			}),
+		).
+		Watches(
+			&ciliumv2.CiliumEgressGatewayPolicy{},
+			handler.EnqueueRequestsFromMapFunc(r.findObjectsForHaegressGatewayPolicy),
+			builder.WithPredicates(predicate.Funcs{
+				DeleteFunc: func(e event.DeleteEvent) bool {
+					return true
+				},
+				CreateFunc: func(e event.CreateEvent) bool {
+					return false
+				},
+				UpdateFunc: func(e event.UpdateEvent) bool {
+					return false
+				},
+				GenericFunc: func(e event.GenericEvent) bool {
+					return false
+				},
+			}),
+		).
 		Complete(r)
 }
diff --git a/controllers/services_controller.go b/controllers/services_controller.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	haegressip "github.com/angeloxx/cilium-haegress-operator/pkg"
+	haegressiputil "github.com/angeloxx/cilium-haegress-operator/util"
 	"github.com/cilium/cilium/pkg/hubble/relay/defaults"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/go-logr/logr"
@@ -70,43 +71,8 @@ func (r *ServicesController) Reconcile(ctx context.Context, req ctrl.Request) (c
 		}
 	}
 
-	policyHost := string(ciliumEgressGatewayPolicy.Spec.EgressGateway.NodeSelector.MatchLabels[haegressip.NodeNameAnnotation])
-	currentHost := string(service.Annotations[haegressip.KubeVIPVipHostAnnotation])
+	return haegressiputil.SyncServiceWithCiliumEgressGatewayPolicy(ctx, r.Client, logger, r.Recorder, service, *ciliumEgressGatewayPolicy)
 
-	if len(service.Status.LoadBalancer.Ingress) > 0 {
-		if ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP != service.Status.LoadBalancer.Ingress[0].IP {
-			ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP = service.Status.LoadBalancer.Ingress[0].IP
-			if err := r.Update(ctx, ciliumEgressGatewayPolicy); err != nil {
-				logger.Error(err, "unable to update the CiliumEgressGatewayPolicy with new assigned IP, retry later")
-				return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, nil
-			}
-			logger.Info("Updated CiliumEgressGatewayPolicy with LoadBalancerIP", "LoadBalancerIP", service.Status.LoadBalancer.Ingress[0].IP)
-		}
-	}
-
-	if currentHost == "" {
-		logger.V(1).Info(fmt.Sprintf("Service is still not assigned, ignoring."))
-		return ctrl.Result{}, nil
-	}
-
-	if policyHost == currentHost {
-		logger.V(1).Info(fmt.Sprintf("EgressGatewayPolicy already configured as expected with host %s, ignoring.", currentHost))
-		return ctrl.Result{}, nil
-	}
-
-	logger.V(0).Info(fmt.Sprintf("EgressGatewayPolicy should be updated from %s to %s.", policyHost, currentHost))
-
-	// Modify egressPolicy nodeSelector to match the service
-	patchData := fmt.Sprintf(`{"spec":{"egressGateway":{"nodeSelector":{"matchLabels":{"%s":"%s"}}}}}`, haegressip.NodeNameAnnotation, currentHost)
-
-	logger.V(0).Info(fmt.Sprintf("Patching cilium egress gateway policy %s with host %s", ciliumEgressGatewayPolicy.Name, currentHost))
-	if err := r.Patch(ctx, ciliumEgressGatewayPolicy, client.RawPatch(types.MergePatchType, []byte(patchData))); err != nil {
-		logger.V(0).Info(fmt.Sprintf("Unable to patch cilium egress gateway policy %s", ciliumEgressGatewayPolicy.Name))
-		return ctrl.Result{RequeueAfter: haegressip.LeaseCheckRequeueAfter}, err
-	}
-	r.Recorder.Event(ciliumEgressGatewayPolicy, "Normal", haegressip.EventEgressUpdateReason, fmt.Sprintf("Updated with new nodeSelector %s=%s by %s/%s service", haegressip.NodeNameAnnotation, currentHost, req.Namespace, req.Name))
-
-	return ctrl.Result{}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.

diff --git a/pkg/types.go b/pkg/types.go
@@ -3,13 +3,12 @@ package haegressip
 import "time"
 
 const (
-	HAEgressGatewayPolicyNamespace         = "cilium.angeloxx.ch/haegressgatewaypolicy-namespace"
-	HAEgressGatewayPolicyName              = "cilium.angeloxx.ch/haegressgatewaypolicy-name"
-	HAEgressGatewayPolicyExpectedLeaseName = "cilium.angeloxx.ch/lease-name"
-	NodeNameAnnotation                     = "kubernetes.io/hostname"
-	EventEgressUpdateReason                = "Updated"
-
-	KubeVIPVipHostAnnotation = "kube-vip.io/vipHost"
+	HAEgressGatewayPolicyNamespace       = "cilium.angeloxx.ch/haegressgatewaypolicy-namespace"
+	HAEgressGatewayPolicyName            = "cilium.angeloxx.ch/haegressgatewaypolicy-name"
+	NodeNameAnnotation                   = "kubernetes.io/hostname"
+	EventEgressUpdateReason              = "Updated"
+	KubeVIPVipHostAnnotation             = "kube-vip.io/vipHost"
+	KubernetesServiceProxyNameAnnotation = "service.kubernetes.io/service-proxy-name"
 
 	LeaseCheckRequeueAfter                 = 10 * time.Second
 	HAEgressGatewayPolicyChcekRequeueAfter = 10 * time.Second