Feat https fallback (#1020)

alibaba · May 31, 2024 · 2807ddf · 2807ddf
1 parent 6e4ade0
commit 2807ddf
Show file tree

Hide file tree

Showing 9 changed files with 215 additions and 56 deletions.
diff --git a/go.mod b/go.mod
@@ -259,7 +259,6 @@ require (
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
@@ -281,6 +280,8 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	k8s.io/apiserver v0.22.5 // indirect
+	k8s.io/component-base v0.22.5 // indirect
+	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
 	oras.land/oras-go v0.4.0 // indirect
@@ -312,10 +313,9 @@ require (
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mholt/acmez v1.2.0
 	github.com/tidwall/gjson v1.17.0
+	golang.org/x/net v0.17.0
 	helm.sh/helm/v3 v3.7.1
 	k8s.io/apiextensions-apiserver v0.25.4
-	k8s.io/component-base v0.22.5
-	k8s.io/klog/v2 v2.60.1
 	knative.dev/networking v0.0.0-20220302134042-e8b2eb995165
 	knative.dev/pkg v0.0.0-20220301181942-2fdd5f232e77
 )

diff --git a/pkg/cert/config.go b/pkg/cert/config.go
@@ -45,11 +45,12 @@ const (
 
 // Config is the configuration of automatic https.
 type Config struct {
-	AutomaticHttps   bool              `json:"automaticHttps"`
-	RenewBeforeDays  int               `json:"renewBeforeDays"`
-	CredentialConfig []CredentialEntry `json:"credentialConfig"`
-	ACMEIssuer       []ACMEIssuerEntry `json:"acmeIssuer"`
-	Version          string            `json:"version"`
+	AutomaticHttps           bool              `json:"automaticHttps"`
+	FallbackForInvalidSecret bool              `json:"fallbackForInvalidSecret"`
+	RenewBeforeDays          int               `json:"renewBeforeDays"`
+	CredentialConfig         []CredentialEntry `json:"credentialConfig"`
+	ACMEIssuer               []ACMEIssuerEntry `json:"acmeIssuer"`
+	Version                  string            `json:"version"`
 }
 
 func (c *Config) GetIssuer(issuerName IssuerName) *ACMEIssuerEntry {
@@ -274,11 +275,12 @@ func newDefaultConfig(email string) *Config {
 	}
 	defaultCredentialConfig := make([]CredentialEntry, 0)
 	config := &Config{
-		AutomaticHttps:   true,
-		RenewBeforeDays:  DefaultRenewBeforeDays,
-		ACMEIssuer:       defaultIssuer,
-		CredentialConfig: defaultCredentialConfig,
-		Version:          time.Now().Format("20060102030405"),
+		AutomaticHttps:           true,
+		FallbackForInvalidSecret: false,
+		RenewBeforeDays:          DefaultRenewBeforeDays,
+		ACMEIssuer:               defaultIssuer,
+		CredentialConfig:         defaultCredentialConfig,
+		Version:                  time.Now().Format("20060102030405"),
 	}
 	return config
 }

diff --git a/pkg/ingress/config/ingress_config.go b/pkg/ingress/config/ingress_config.go
@@ -51,6 +51,7 @@ import (
 	higressv1 "github.com/alibaba/higress/api/networking/v1"
 	extlisterv1 "github.com/alibaba/higress/client/pkg/listers/extensions/v1alpha1"
 	netlisterv1 "github.com/alibaba/higress/client/pkg/listers/networking/v1"
+	"github.com/alibaba/higress/pkg/cert"
 	"github.com/alibaba/higress/pkg/ingress/kube/annotations"
 	"github.com/alibaba/higress/pkg/ingress/kube/common"
 	"github.com/alibaba/higress/pkg/ingress/kube/configmap"
@@ -144,6 +145,8 @@ type IngressConfig struct {
 	namespace string
 
 	clusterId string
+
+	httpsConfigMgr *cert.ConfigMgr
 }
 
 func NewIngressConfig(localKubeClient kube.Client, XDSUpdater model.XDSUpdater, namespace, clusterId string) *IngressConfig {
@@ -180,6 +183,9 @@ func NewIngressConfig(localKubeClient kube.Client, XDSUpdater model.XDSUpdater,
 	higressConfigController := configmap.NewController(localKubeClient, clusterId, namespace)
 	config.configmapMgr = configmap.NewConfigmapMgr(XDSUpdater, namespace, higressConfigController, higressConfigController.Lister())
 
+	httpsConfigMgr, _ := cert.NewConfigMgr(namespace, localKubeClient)
+	config.httpsConfigMgr = httpsConfigMgr
+
 	return config
 }
 
@@ -347,6 +353,10 @@ func (m *IngressConfig) convertGateways(configs []common.WrapperConfig) []config
 		Gateways:           map[string]*common.WrapperGateway{},
 	}
 
+	httpsCredentialConfig, err := m.httpsConfigMgr.GetConfigFromConfigmap()
+	if err != nil {
+		IngressLog.Errorf("Get higress https configmap err %v", err)
+	}
 	for idx := range configs {
 		cfg := configs[idx]
 		clusterId := common.GetClusterId(cfg.Config.Annotations)
@@ -356,7 +366,7 @@ func (m *IngressConfig) convertGateways(configs []common.WrapperConfig) []config
 		if ingressController == nil {
 			continue
 		}
-		if err := ingressController.ConvertGateway(&convertOptions, &cfg); err != nil {
+		if err := ingressController.ConvertGateway(&convertOptions, &cfg, httpsCredentialConfig); err != nil {
 			IngressLog.Errorf("Convert ingress %s/%s to gateway fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err)
 		}
 	}

diff --git a/pkg/ingress/kube/common/controller.go b/pkg/ingress/kube/common/controller.go
@@ -17,14 +17,14 @@ package common
 import (
 	"strings"
 
+	"github.com/alibaba/higress/pkg/cert"
+	"github.com/alibaba/higress/pkg/ingress/kube/annotations"
 	networking "istio.io/api/networking/v1alpha3"
 	"istio.io/istio/pilot/pkg/model"
 	"istio.io/istio/pkg/config"
 	gatewaytool "istio.io/istio/pkg/config/gateway"
 	listerv1 "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
-
-	"github.com/alibaba/higress/pkg/ingress/kube/annotations"
 )
 
 type ServiceKey struct {
@@ -121,7 +121,7 @@ type IngressController interface {
 
 	SecretLister() listerv1.SecretLister
 
-	ConvertGateway(convertOptions *ConvertOptions, wrapper *WrapperConfig) error
+	ConvertGateway(convertOptions *ConvertOptions, wrapper *WrapperConfig, httpsCredentialConfig *cert.Config) error
 
 	ConvertHTTPRoute(convertOptions *ConvertOptions, wrapper *WrapperConfig) error
 

diff --git a/pkg/ingress/kube/ingress/controller.go b/pkg/ingress/kube/ingress/controller.go
@@ -55,6 +55,7 @@ import (
 	"github.com/alibaba/higress/pkg/ingress/kube/secret"
 	"github.com/alibaba/higress/pkg/ingress/kube/util"
 	. "github.com/alibaba/higress/pkg/ingress/log"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 var (
@@ -87,8 +88,6 @@ type controller struct {
 	secretController secret.SecretController
 
 	statusSyncer *statusSyncer
-
-	configMgr *cert.ConfigMgr
 }
 
 // NewController creates a new Kubernetes controller
@@ -107,7 +106,6 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
 		IngressLog.Infof("Skipping IngressClass, resource not supported for cluster %s", options.ClusterId)
 	}
 
-	configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
 	c := &controller{
 		options:          options,
 		queue:            q,
@@ -118,7 +116,6 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
 		serviceInformer:  serviceInformer.Informer(),
 		serviceLister:    serviceInformer.Lister(),
 		secretController: secretController,
-		configMgr:        configMgr,
 	}
 
 	handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
@@ -354,7 +351,7 @@ func extractTLSSecretName(host string, tls []ingress.IngressTLS) string {
 	return ""
 }
 
-func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error {
+func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig, httpsCredentialConfig *cert.Config) error {
 	if convertOptions == nil {
 		return fmt.Errorf("convertOptions is nil")
 	}
@@ -377,7 +374,6 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
 		common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule)
 		return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
 	}
-	httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
 	for _, rule := range ingressV1Beta.Rules {
 		// Need create builder for every rule.
 		domainBuilder := &common.IngressDomainBuilder{
@@ -429,10 +425,23 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
 		// Get tls secret matching the rule host
 		secretName := extractTLSSecretName(rule.Host, ingressV1Beta.TLS)
 		secretNamespace := cfg.Namespace
-		// If there is no matching secret, try to get it from configmap.
-		if secretName == "" && httpsCredentialConfig != nil {
-			secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
-			secretNamespace = c.options.SystemNamespace
+		if secretName != "" {
+			if httpsCredentialConfig != nil && httpsCredentialConfig.FallbackForInvalidSecret {
+				_, err := c.secretController.Lister().Secrets(secretNamespace).Get(secretName)
+				if err != nil {
+					if k8serrors.IsNotFound(err) {
+						// If there is no matching secret, try to get it from configmap.
+						secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
+						secretNamespace = c.options.SystemNamespace
+					}
+				}
+			}
+		} else {
+			// If there is no matching secret, try to get it from configmap.
+			if httpsCredentialConfig != nil {
+				secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
+				secretNamespace = c.options.SystemNamespace
+			}
 		}
 		if secretName == "" {
 			// There no matching secret, so just skip.

diff --git a/pkg/ingress/kube/ingress/controller_test.go b/pkg/ingress/kube/ingress/controller_test.go
@@ -334,7 +334,7 @@ func testConvertGateway(t *testing.T, c common.IngressController) {
 	}
 
 	for _, testcase := range testcases {
-		err := c.ConvertGateway(testcase.input.options, testcase.input.wrapperConfig)
+		err := c.ConvertGateway(testcase.input.options, testcase.input.wrapperConfig, nil)
 		if err != nil {
 			require.Equal(t, testcase.expectNoError, false)
 		} else {

diff --git a/pkg/ingress/kube/ingressv1/controller.go b/pkg/ingress/kube/ingressv1/controller.go
@@ -54,6 +54,7 @@ import (
 	"github.com/alibaba/higress/pkg/ingress/kube/secret"
 	"github.com/alibaba/higress/pkg/ingress/kube/util"
 	. "github.com/alibaba/higress/pkg/ingress/log"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 var (
@@ -85,8 +86,6 @@ type controller struct {
 	secretController secret.SecretController
 
 	statusSyncer *statusSyncer
-
-	configMgr *cert.ConfigMgr
 }
 
 // NewController creates a new Kubernetes controller
@@ -99,7 +98,6 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
 	classes := client.KubeInformer().Networking().V1().IngressClasses()
 	classes.Informer()
 
-	configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
 	c := &controller{
 		options:          options,
 		queue:            q,
@@ -110,7 +108,6 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
 		serviceInformer:  serviceInformer.Informer(),
 		serviceLister:    serviceInformer.Lister(),
 		secretController: secretController,
-		configMgr:        configMgr,
 	}
 
 	handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
@@ -346,7 +343,7 @@ func extractTLSSecretName(host string, tls []ingress.IngressTLS) string {
 	return ""
 }
 
-func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error {
+func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig, httpsCredentialConfig *cert.Config) error {
 	// Ignore canary config.
 	if wrapper.AnnotationsConfig.IsCanary() {
 		return nil
@@ -363,7 +360,6 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
 		return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
 	}
 
-	httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
 	for _, rule := range ingressV1.Rules {
 		// Need create builder for every rule.
 		domainBuilder := &common.IngressDomainBuilder{
@@ -415,11 +411,25 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
 		// Get tls secret matching the rule host
 		secretName := extractTLSSecretName(rule.Host, ingressV1.TLS)
 		secretNamespace := cfg.Namespace
-		// If there is no matching secret, try to get it from configmap.
-		if secretName == "" && httpsCredentialConfig != nil {
-			secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
-			secretNamespace = c.options.SystemNamespace
+		if secretName != "" {
+			if httpsCredentialConfig != nil && httpsCredentialConfig.FallbackForInvalidSecret {
+				_, err := c.secretController.Lister().Secrets(secretNamespace).Get(secretName)
+				if err != nil {
+					if k8serrors.IsNotFound(err) {
+						// If there is no matching secret, try to get it from configmap.
+						secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
+						secretNamespace = c.options.SystemNamespace
+					}
+				}
+			}
+		} else {
+			// If there is no matching secret, try to get it from configmap.
+			if httpsCredentialConfig != nil {
+				secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
+				secretNamespace = c.options.SystemNamespace
+			}
 		}
+
 		if secretName == "" {
 			// There no matching secret, so just skip.
 			continue