From 7e800dde1cbc53fcbfdc3c1c8309f3a832b5a182 Mon Sep 17 00:00:00 2001 From: Andrei Nistor Date: Wed, 27 Apr 2022 09:49:42 +0300 Subject: [PATCH] Fix resource consumption issues (#125) * Use varnishreload-compatible vcl names https://github.com/mittwald/kube-httpcache/issues/81#issuecomment-883296355 * Honor varnishd max_vcl https://varnish-cache.org/docs/6.0/reference/varnishd.html#max-vcl --- go.mod | 2 +- go.sum | 4 +++ pkg/controller/watch.go | 57 ++++++++++++++++++++++++++++++++++------- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index b23d666..37edeb7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/json-iterator/go v1.1.11 // indirect - github.com/martin-helmich/go-varnish-client v0.2.1 + github.com/martin-helmich/go-varnish-client v0.2.2 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect diff --git a/go.sum b/go.sum index 3c322bc..35f4696 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/martin-helmich/go-varnish-client v0.2.0 h1:UNeB/8PQ+inkn1v57KLg2s3UkB github.com/martin-helmich/go-varnish-client v0.2.0/go.mod h1:BPZwupDItHsp9jLWUCFtchhJoJlt1RIjWoPEfeWTRdI= github.com/martin-helmich/go-varnish-client v0.2.1 h1:+NTZ+Io0dXMVrLJ534ABt8zr34sZop1fjt683qybmhg= github.com/martin-helmich/go-varnish-client v0.2.1/go.mod h1:mTUoKZn5fbP67YKPfKP2Zi8Y6A/+99B0oCQBFuHOhks= +github.com/martin-helmich/go-varnish-client v0.2.2 h1:WBW/SwBPX+SEJN2nKh1/mhdgRqdNLgndxFE4RyD51hE= +github.com/martin-helmich/go-varnish-client v0.2.2/go.mod h1:BUVEoli2BmESGMvlRiytvurkdZhJQwlzFtTBmG9mfFA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -126,6 +128,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490 h1:va0qYsIOza3Nlf2IncFyOql4/3XUq3vfge/Ad64bhlM= golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -194,6 +197,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.0.0-20181013054003-e94254e9898f h1:kLihNpdVw0H7RpDrS0XH9yI0+TI90p6YbaKlitJkaPc= k8s.io/api v0.0.0-20181013054003-e94254e9898f/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20181013010248-dcb88206cd7f h1:J4RMsuKRhuF+JxWL8Ip+w+lVxS/kypd+j0R93qPBD2c= diff --git a/pkg/controller/watch.go b/pkg/controller/watch.go index b59a945..7b0859a 100644 --- a/pkg/controller/watch.go +++ b/pkg/controller/watch.go @@ -5,18 +5,18 @@ import ( "context" "fmt" "os/exec" + "sort" + "strconv" + "strings" "text/template" + "time" "github.com/golang/glog" varnishclient "github.com/martin-helmich/go-varnish-client" ) func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, errors chan<- error) { - i := 0 - for { - i++ - select { case tmplContents := <-v.vclTemplateUpdates: glog.Infof("VCL template has been updated") @@ -29,7 +29,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, v.vclTemplate = tmpl - errors <- v.rebuildConfig(ctx, i) + errors <- v.rebuildConfig(ctx) case newConfig := <-v.frontendUpdates: glog.Infof("received new frontend configuration: %+v", newConfig) @@ -40,14 +40,14 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, v.varnishSignaller.SetEndpoints(v.frontend) } - errors <- v.rebuildConfig(ctx, i) + errors <- v.rebuildConfig(ctx) case newConfig := <-v.backendUpdates: glog.Infof("received new backend configuration: %+v", newConfig) v.backend = newConfig - errors <- v.rebuildConfig(ctx, i) + errors <- v.rebuildConfig(ctx) case <-ctx.Done(): errors <- ctx.Err() @@ -56,7 +56,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, } } -func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error { +func (v *VarnishController) rebuildConfig(ctx context.Context) error { buf := new(bytes.Buffer) err := v.renderVCL(buf, v.frontend.Endpoints, v.frontend.Primary, v.backend.Endpoints, v.backend.Primary) @@ -77,7 +77,46 @@ func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error { return err } - configname := fmt.Sprintf("k8s-upstreamcfg-%d", i) + maxVclParam, err := client.GetParameter(ctx, "max_vcl") + if err != nil { + return err + } + + maxVcl, err := strconv.Atoi(maxVclParam.Value) + if err != nil { + return err + } + + loadedVcl, err := client.ListVCL(ctx) + if err != nil { + return err + } + + availableVcl := make([]varnishclient.VCLConfig, 0) + + for i := range loadedVcl { + if loadedVcl[i].Status == varnishclient.VCLAvailable { + availableVcl = append(availableVcl, loadedVcl[i]) + } + } + + if len(loadedVcl) >= maxVcl { + // we're abusing the fact that "boot" < "reload" + sort.Slice(availableVcl, func(i, j int) bool { + return availableVcl[i].Name < availableVcl[j].Name + }) + + for i := 0; i < len(loadedVcl)-maxVcl+1; i++ { + glog.V(8).Infof("discarding VCL: %s", availableVcl[i].Name) + + err = client.DiscardVCL(ctx, availableVcl[i].Name) + if err != nil { + return err + } + } + } + + configname := strings.ReplaceAll(time.Now().Format("reload_20060102_150405.00000"), ".", "_") err = client.DefineInlineVCL(ctx, configname, vcl, varnishclient.VCLStateAuto) if err != nil {