From 00cdd079392f31724745d112576e331058454ace Mon Sep 17 00:00:00 2001 From: Angelo Conforti Date: Sun, 1 Sep 2024 14:48:40 +0200 Subject: [PATCH] add detailed status, new CRD --- api/v2/haegressgatewaypolicy_types.go | 13 ++- .../templates/crds/haegressgatewaypolicy.yaml | 18 ++-- ...m.angeloxx.ch_haegressgatewaypolicies.yaml | 18 ++-- .../haegressgatewaypolicies_controller.go | 102 +++++++++++++----- controllers/services_controller.go | 38 +------ pkg/types.go | 13 ++- util/util.go | 93 ++++++++++++++++ 7 files changed, 210 insertions(+), 85 deletions(-) create mode 100644 util/util.go diff --git a/api/v2/haegressgatewaypolicy_types.go b/api/v2/haegressgatewaypolicy_types.go index 8e68500..b1ddbd5 100644 --- a/api/v2/haegressgatewaypolicy_types.go +++ b/api/v2/haegressgatewaypolicy_types.go @@ -22,20 +22,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy type HAEgressGatewayPolicyStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file ServiceCreated bool `json:"serviceCreated"` PolicyCreated bool `json:"policyCreated"` + + // +kubebuilder:validation:Optional + ExitNode string `json:"exitNode,omitempty"` + + // +kubebuilder:validation:Optional + IPAddress string `json:"ipAddress,omitempty"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status //+kubebuilder:resource:scope=Cluster +//+kubebuilder:printcolumn:name="IP Address",type=string,JSONPath=`.status.ipAddress` +//+kubebuilder:printcolumn:name="Exit Node",type=string,JSONPath=`.status.exitNode` // haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies API type HAEgressGatewayPolicy struct { diff --git a/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml b/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml index 419722e..f0ecba6 100644 --- a/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml +++ b/charts/cilium-ha-egress/templates/crds/haegressgatewaypolicy.yaml @@ -12,11 +12,16 @@ spec: listKind: HAEgressGatewayPolicyList plural: haegressgatewaypolicies singular: haegressgatewaypolicy - shortNames: - - haegp scope: Cluster versions: - - name: v2 + - additionalPrinterColumns: + - jsonPath: .status.ipAddress + name: IP Address + type: string + - jsonPath: .status.exitNode + name: Exit Node + type: string + name: v2 schema: openAPIV3Schema: description: haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies @@ -253,12 +258,13 @@ spec: status: description: HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy properties: + exitNode: + type: string + ipAddress: + type: string policyCreated: type: boolean serviceCreated: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state - of cluster Important: Run "make" to regenerate code after modifying - this file' type: boolean required: - policyCreated diff --git a/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml b/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml index f8ff454..72ca52f 100644 --- a/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml +++ b/config/crd/bases/cilium.angeloxx.ch_haegressgatewaypolicies.yaml @@ -12,11 +12,16 @@ spec: listKind: HAEgressGatewayPolicyList plural: haegressgatewaypolicies singular: haegressgatewaypolicy - shortNames: - - haegp scope: Cluster versions: - - name: v2 + - additionalPrinterColumns: + - jsonPath: .status.ipAddress + name: IP Address + type: string + - jsonPath: .status.exitNode + name: Exit Node + type: string + name: v2 schema: openAPIV3Schema: description: haEgressGatewayPolicy is the Schema for the haegressgatewaypolicies @@ -253,12 +258,13 @@ spec: status: description: HAEgressGatewayPolicy defines the observed state of haEgressGatewayPolicy properties: + exitNode: + type: string + ipAddress: + type: string policyCreated: type: boolean serviceCreated: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state - of cluster Important: Run "make" to regenerate code after modifying - this file' type: boolean required: - policyCreated diff --git a/controllers/haegressgatewaypolicies_controller.go b/controllers/haegressgatewaypolicies_controller.go index 36aad2e..178ac47 100644 --- a/controllers/haegressgatewaypolicies_controller.go +++ b/controllers/haegressgatewaypolicies_controller.go @@ -21,6 +21,7 @@ import ( "fmt" haegressv2 "github.com/angeloxx/cilium-haegress-operator/api/v2" haegressip "github.com/angeloxx/cilium-haegress-operator/pkg" + haegressiputil "github.com/angeloxx/cilium-haegress-operator/util" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -31,8 +32,13 @@ import ( "k8s.io/client-go/tools/record" "reflect" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // HAEgressGatewayPolicyReconciler reconciles a HAEgressGatewayPolicy object @@ -74,23 +80,6 @@ func (r *HAEgressGatewayPolicyReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, err } - /* - serviceNamespace := r.EgressNamespace - if haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] != "" { - serviceNamespace = haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] - } - leaseExpectedName := fmt.Sprintf("cilium-l2announce-%s-%s", - serviceNamespace, haEgressGatewayPolicy.Name) - if haEgressGatewayPolicy.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] != leaseExpectedName { - haEgressGatewayPolicy.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] = leaseExpectedName - - if err := r.Update(ctx, &haEgressGatewayPolicy); err != nil { - log.Error(err, "unable to update HAEgressGatewayPolicy, please check RBAC permissions") - return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, err - } - } - - */ if err := r.UpdateOrCreateCiliumEgressGatewayPolicy(ctx, &haEgressGatewayPolicy); err != nil { log.Error(err, "unable to create or update CiliumEgressGatewayPolicy, please check RBAC permissions") return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, err @@ -113,11 +102,6 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateCiliumEgressGatewayPolic if haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] != "" { serviceNamespace = haEgressGatewayPolicy.Annotations[haegressip.HAEgressGatewayPolicyNamespace] } - /* - leaseExpectedName := fmt.Sprintf("cilium-l2announce-%s-%s", - serviceNamespace, haEgressGatewayPolicy.Name) - ciliumEgressGatewayPolicyNew.Labels[haegressip.HAEgressGatewayPolicyExpectedLeaseName] = leaseExpectedName - */ ciliumEgressGatewayPolicyNew := &ciliumv2.CiliumEgressGatewayPolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -154,6 +138,18 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateCiliumEgressGatewayPolic if err := controllerutil.SetControllerReference(haEgressGatewayPolicy, ciliumEgressGatewayPolicyNew, r.Scheme); err != nil { return err } + + // If service already exists, reconcile + service := &corev1.Service{} + err = r.Get(ctx, types.NamespacedName{Name: haEgressGatewayPolicy.Name, Namespace: serviceNamespace}, service) + if err == nil { + // Call the services reconcile function + _, syncError := haegressiputil.SyncServiceWithCiliumEgressGatewayPolicy(ctx, r.Client, logger, r.Recorder, *service, *ciliumEgressGatewayPolicyNew) + if syncError != nil { + return syncError + } + } + } else if err != nil { return err } else { @@ -226,7 +222,7 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont service.Annotations = make(map[string]string) } // Avoid L2 announcement by Cilium - service.Labels["service.kubernetes.io/service-proxy-name"] = "kubevip-managed-by-cilium-haegess" + service.Labels[haegressip.KubernetesServiceProxyNameAnnotation] = "kubevip-managed-by-cilium-haegess" service.Labels[haegressip.HAEgressGatewayPolicyNamespace] = serviceNamespace service.Labels[haegressip.HAEgressGatewayPolicyName] = haEgressGatewayPolicy.Name @@ -241,8 +237,10 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont if err != nil && apierrors.IsNotFound(err) { log.Info("Creating a new Service for HAEgressGatewayPolicy", "Service.Namespace", service.Namespace, "Service.Name", service.Name) err = r.Create(ctx, service) - r.Recorder.Event(haEgressGatewayPolicy, corev1.EventTypeNormal, "Created", "Service created") - + r.Recorder.Event(haEgressGatewayPolicy, + corev1.EventTypeNormal, + "Created", + fmt.Sprintf("Service %s/%s created", service.Namespace, service.Name)) if err != nil { return err } @@ -271,9 +269,63 @@ func (r *HAEgressGatewayPolicyReconciler) UpdateOrCreateService(ctx context.Cont return nil } +func (r *HAEgressGatewayPolicyReconciler) findObjectsForHaegressGatewayPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + ownerRefs := obj.GetOwnerReferences() + requests := []reconcile.Request{} + + for _, ownerRef := range ownerRefs { + if ownerRef.Kind == "HAEgressGatewayPolicy" { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: ownerRef.Name, + Namespace: obj.GetNamespace(), + }, + }) + } + } + + return requests +} + // SetupWithManager sets up the controller with the Manager. func (r *HAEgressGatewayPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&haegressv2.HAEgressGatewayPolicy{}). + Watches( + &corev1.Service{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForHaegressGatewayPolicy), + builder.WithPredicates(predicate.Funcs{ + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). + Watches( + &ciliumv2.CiliumEgressGatewayPolicy{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForHaegressGatewayPolicy), + builder.WithPredicates(predicate.Funcs{ + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + }), + ). Complete(r) } diff --git a/controllers/services_controller.go b/controllers/services_controller.go index 2ea36eb..b605c52 100644 --- a/controllers/services_controller.go +++ b/controllers/services_controller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" haegressip "github.com/angeloxx/cilium-haegress-operator/pkg" + haegressiputil "github.com/angeloxx/cilium-haegress-operator/util" "github.com/cilium/cilium/pkg/hubble/relay/defaults" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" "github.com/go-logr/logr" @@ -70,43 +71,8 @@ func (r *ServicesController) Reconcile(ctx context.Context, req ctrl.Request) (c } } - policyHost := string(ciliumEgressGatewayPolicy.Spec.EgressGateway.NodeSelector.MatchLabels[haegressip.NodeNameAnnotation]) - currentHost := string(service.Annotations[haegressip.KubeVIPVipHostAnnotation]) + return haegressiputil.SyncServiceWithCiliumEgressGatewayPolicy(ctx, r.Client, logger, r.Recorder, service, *ciliumEgressGatewayPolicy) - if len(service.Status.LoadBalancer.Ingress) > 0 { - if ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP != service.Status.LoadBalancer.Ingress[0].IP { - ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP = service.Status.LoadBalancer.Ingress[0].IP - if err := r.Update(ctx, ciliumEgressGatewayPolicy); err != nil { - logger.Error(err, "unable to update the CiliumEgressGatewayPolicy with new assigned IP, retry later") - return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, nil - } - logger.Info("Updated CiliumEgressGatewayPolicy with LoadBalancerIP", "LoadBalancerIP", service.Status.LoadBalancer.Ingress[0].IP) - } - } - - if currentHost == "" { - logger.V(1).Info(fmt.Sprintf("Service is still not assigned, ignoring.")) - return ctrl.Result{}, nil - } - - if policyHost == currentHost { - logger.V(1).Info(fmt.Sprintf("EgressGatewayPolicy already configured as expected with host %s, ignoring.", currentHost)) - return ctrl.Result{}, nil - } - - logger.V(0).Info(fmt.Sprintf("EgressGatewayPolicy should be updated from %s to %s.", policyHost, currentHost)) - - // Modify egressPolicy nodeSelector to match the service - patchData := fmt.Sprintf(`{"spec":{"egressGateway":{"nodeSelector":{"matchLabels":{"%s":"%s"}}}}}`, haegressip.NodeNameAnnotation, currentHost) - - logger.V(0).Info(fmt.Sprintf("Patching cilium egress gateway policy %s with host %s", ciliumEgressGatewayPolicy.Name, currentHost)) - if err := r.Patch(ctx, ciliumEgressGatewayPolicy, client.RawPatch(types.MergePatchType, []byte(patchData))); err != nil { - logger.V(0).Info(fmt.Sprintf("Unable to patch cilium egress gateway policy %s", ciliumEgressGatewayPolicy.Name)) - return ctrl.Result{RequeueAfter: haegressip.LeaseCheckRequeueAfter}, err - } - r.Recorder.Event(ciliumEgressGatewayPolicy, "Normal", haegressip.EventEgressUpdateReason, fmt.Sprintf("Updated with new nodeSelector %s=%s by %s/%s service", haegressip.NodeNameAnnotation, currentHost, req.Namespace, req.Name)) - - return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. diff --git a/pkg/types.go b/pkg/types.go index 00acee0..e12c679 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -3,13 +3,12 @@ package haegressip import "time" const ( - HAEgressGatewayPolicyNamespace = "cilium.angeloxx.ch/haegressgatewaypolicy-namespace" - HAEgressGatewayPolicyName = "cilium.angeloxx.ch/haegressgatewaypolicy-name" - HAEgressGatewayPolicyExpectedLeaseName = "cilium.angeloxx.ch/lease-name" - NodeNameAnnotation = "kubernetes.io/hostname" - EventEgressUpdateReason = "Updated" - - KubeVIPVipHostAnnotation = "kube-vip.io/vipHost" + HAEgressGatewayPolicyNamespace = "cilium.angeloxx.ch/haegressgatewaypolicy-namespace" + HAEgressGatewayPolicyName = "cilium.angeloxx.ch/haegressgatewaypolicy-name" + NodeNameAnnotation = "kubernetes.io/hostname" + EventEgressUpdateReason = "Updated" + KubeVIPVipHostAnnotation = "kube-vip.io/vipHost" + KubernetesServiceProxyNameAnnotation = "service.kubernetes.io/service-proxy-name" LeaseCheckRequeueAfter = 10 * time.Second HAEgressGatewayPolicyChcekRequeueAfter = 10 * time.Second diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..baecb8a --- /dev/null +++ b/util/util.go @@ -0,0 +1,93 @@ +package util + +import ( + "context" + "fmt" + v2 "github.com/angeloxx/cilium-haegress-operator/api/v2" + haegressip "github.com/angeloxx/cilium-haegress-operator/pkg" + ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func SyncServiceWithCiliumEgressGatewayPolicy(ctx context.Context, r client.Client, logger logr.Logger, recorder record.EventRecorder, service corev1.Service, ciliumEgressGatewayPolicy ciliumv2.CiliumEgressGatewayPolicy) (ctrl.Result, error) { + + // Get the parent HAEgressGatewayPolicy from the ciliumEgressGatewayPolicy + haEgressGatewayPolicy := &v2.HAEgressGatewayPolicy{} + ownerRefs := ciliumEgressGatewayPolicy.GetOwnerReferences() + for _, ownerRef := range ownerRefs { + if ownerRef.Kind == "HAEgressGatewayPolicy" { + if err := r.Get(ctx, types.NamespacedName{Name: ownerRef.Name, Namespace: ciliumEgressGatewayPolicy.Namespace}, haEgressGatewayPolicy); err != nil { + logger.Error(err, "unable to fetch the HAEgressGatewayPolicy, check RBAC permissions") + return ctrl.Result{}, nil + } + break + } + } + + policyHost := string(ciliumEgressGatewayPolicy.Spec.EgressGateway.NodeSelector.MatchLabels[haegressip.NodeNameAnnotation]) + currentHost := string(service.Annotations[haegressip.KubeVIPVipHostAnnotation]) + + if len(service.Status.LoadBalancer.Ingress) > 0 { + if ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP != service.Status.LoadBalancer.Ingress[0].IP { + ciliumEgressGatewayPolicy.Spec.EgressGateway.EgressIP = service.Status.LoadBalancer.Ingress[0].IP + if err := r.Update(ctx, &ciliumEgressGatewayPolicy); err != nil { + logger.Error(err, "unable to update the CiliumEgressGatewayPolicy with new assigned IP, retry later") + return ctrl.Result{RequeueAfter: haegressip.HAEgressGatewayPolicyChcekRequeueAfter}, nil + } + logger.Info("Updated CiliumEgressGatewayPolicy with LoadBalancerIP", "LoadBalancerIP", service.Status.LoadBalancer.Ingress[0].IP) + + } + if haEgressGatewayPolicy.Status.IPAddress != service.Status.LoadBalancer.Ingress[0].IP { + haEgressGatewayPolicy.Status.IPAddress = service.Status.LoadBalancer.Ingress[0].IP + if err := r.Status().Update(ctx, haEgressGatewayPolicy); err != nil { + logger.Error(err, "unable to update the HAEgressGatewayPolicy with new assigned IP") + } + } + } + + if currentHost == "" { + logger.V(1).Info(fmt.Sprintf("Service is still not assigned, ignoring.")) + return ctrl.Result{}, nil + } + + if haEgressGatewayPolicy.Status.ExitNode != currentHost { + haEgressGatewayPolicy.Status.ExitNode = currentHost + if err := r.Status().Update(ctx, haEgressGatewayPolicy); err != nil { + logger.Error(err, "unable to update the HAEgressGatewayPolicy with new assigned exitNode") + } + } + + if policyHost == currentHost { + logger.V(1).Info(fmt.Sprintf("EgressGatewayPolicy already configured as expected with host %s, ignoring.", currentHost)) + return ctrl.Result{}, nil + } + + logger.V(0).Info(fmt.Sprintf("EgressGatewayPolicy should be updated from %s to %s.", policyHost, currentHost)) + + // Modify egressPolicy nodeSelector to match the service + patchData := fmt.Sprintf(`{"spec":{"egressGateway":{"nodeSelector":{"matchLabels":{"%s":"%s"}}}}}`, haegressip.NodeNameAnnotation, currentHost) + + logger.V(0).Info(fmt.Sprintf("Patching cilium egress gateway policy %s with host %s", ciliumEgressGatewayPolicy.Name, currentHost)) + if err := r.Patch(ctx, &ciliumEgressGatewayPolicy, client.RawPatch(types.MergePatchType, []byte(patchData))); err != nil { + logger.V(0).Info(fmt.Sprintf("Unable to patch cilium egress gateway policy %s", ciliumEgressGatewayPolicy.Name)) + return ctrl.Result{RequeueAfter: haegressip.LeaseCheckRequeueAfter}, err + } + + recorder.Event(&ciliumEgressGatewayPolicy, "Normal", + haegressip.EventEgressUpdateReason, + fmt.Sprintf("Updated with new nodeSelector %s=%s by %s/%s service", + haegressip.NodeNameAnnotation, currentHost, + service.Namespace, service.Name)) + + recorder.Event(&service, "Normal", + haegressip.EventEgressUpdateReason, + fmt.Sprintf("Updated CiliumEgressGatewayPolicy %s with new nodeSelector %s=%s", + ciliumEgressGatewayPolicy.Name, + haegressip.NodeNameAnnotation, currentHost)) + return ctrl.Result{}, nil +}