feat: stateful sql and folder checks

api/context/context.go

@@ -84,26 +84,27 @@ func getDomain(username string) string {
 	return ""
 }
 
-func (ctx *Context) GetFunctionsFor(check external.Check) map[string]any {
-	env := make(map[string]any)
-
-	return env
-}
-
 func (ctx *Context) Template(check external.Check, template string) (string, error) {
 	env := ctx.Environment
 
-	for k, v := range ctx.GetFunctionsFor(check) {
-		env[k] = v
+	tpl := gomplate.Template{Template: template}
+	if tpl.Functions == nil {
+		tpl.Functions = make(map[string]func() any)
 	}
-
-	out, err := gomplate.RunExpression(env, gomplate.Template{Template: template})
+	for k, v := range ctx.GetContextualFunctions() {
+		tpl.Functions[k] = v
+	}
+	out, err := gomplate.RunExpression(env, tpl)
 	if err != nil {
 		return "", err
 	}
 	return fmt.Sprintf("%s", out), nil
 }
 
+func (ctx *Context) CanTemplate() bool {
+	return ctx.Canary.Annotations["template"] != "false"
+}
+
 func (ctx *Context) GetConnection(conn v1.Connection) (*models.Connection, error) {
 	var _conn *models.Connection
 	var err error
@@ -158,6 +159,21 @@ func (ctx *Context) GetConnection(conn v1.Connection) (*models.Connection, error
 	return _conn, nil
 }
 
+func (ctx Context) TemplateStruct(o interface{}) error {
+	templater := gomplate.StructTemplater{
+		Values: ctx.Environment,
+		Funcs:  ctx.GetContextualFunctions(),
+		// access go values in template requires prefix everything with .
+		// to support $(username) instead of $(.username) we add a function for each var
+		ValueFunctions: true,
+		DelimSets: []gomplate.Delims{
+			{Left: "{{", Right: "}}"},
+			{Left: "$(", Right: ")"},
+		},
+	}
+	return templater.Walk(o)
+}
+
 func (ctx Context) GetAuthValues(auth v1.Authentication) (v1.Authentication, error) {
 	// in case nil we are sending empty string values for username and password
 	if auth.IsEmpty() {

api/context/functions.go

@@ -5,16 +5,13 @@ import (
 	"time"
 
 	"github.com/flanksource/commons/logger"
-	"github.com/flanksource/gomplate/v3"
 )
 
-func (ctx *Context) InjectFunctions(template *gomplate.Template) {
+func (ctx *Context) GetContextualFunctions() map[string]func() any {
+	funcs := make(map[string]func() any)
 	if check, ok := ctx.Environment["check"]; ok {
 		checkID := check.(map[string]any)["id"]
-		if template.Functions == nil {
-			template.Functions = make(map[string]func() any)
-		}
-		template.Functions["last_result"] = func() any {
+		funcs["last_result"] = func() any {
 			if ctx.cache == nil {
 				ctx.cache = make(map[string]any)
 			}
@@ -78,4 +75,5 @@ func (ctx *Context) InjectFunctions(template *gomplate.Template) {
 			return status
 		}
 	}
+	return funcs
 }

api/v1/common.go

@@ -1,6 +1,7 @@
 package v1
 
 import (
+	"fmt"
 	"io/fs"
 	"net/url"
 	"regexp"
@@ -13,6 +14,7 @@ import (
 	"github.com/flanksource/commons/duration"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/gomplate/v3"
+	"github.com/timberio/go-datemath"
 )
 
 type Duration string
@@ -49,16 +51,41 @@ func (s Size) Value() (*int64, error) {
 type FolderFilter struct {
 	MinAge  Duration `yaml:"minAge,omitempty" json:"minAge,omitempty"`
 	MaxAge  Duration `yaml:"maxAge,omitempty" json:"maxAge,omitempty"`
+	Since   string   `yaml:"since,omitempty" json:"since,omitempty"`
 	MinSize Size     `yaml:"minSize,omitempty" json:"minSize,omitempty"`
 	MaxSize Size     `yaml:"maxSize,omitempty" json:"maxSize,omitempty"`
 	Regex   string   `yaml:"regex,omitempty" json:"regex,omitempty"`
 }
 
+func (f FolderFilter) String() string {
+	s := []string{}
+	if f.MinAge != "" {
+		s = append(s, fmt.Sprintf("minAge="+string(f.MinAge)))
+	}
+	if f.MaxAge != "" {
+		s = append(s, "maxAge="+string(f.MaxAge))
+	}
+	if f.MinSize != "" {
+		s = append(s, "minSize="+string(f.MinSize))
+	}
+	if f.MaxSize != "" {
+		s = append(s, "maxSize="+string(f.MaxSize))
+	}
+	if f.Regex != "" {
+		s = append(s, "regex="+f.Regex)
+	}
+	if f.Since != "" {
+		s = append(s, "since="+f.Since)
+	}
+	return strings.Join(s, ", ")
+}
+
 // +k8s:deepcopy-gen=false
 type FolderFilterContext struct {
 	FolderFilter
 	minAge, maxAge   *time.Duration
 	minSize, maxSize *int64
+	Since            *time.Time
 	// kubebuilder:object:generate=false
 	regex *regexp.Regexp
 }
@@ -98,9 +125,36 @@ func (f FolderFilter) New() (*FolderFilterContext, error) {
 			return nil, err
 		}
 	}
+	if f.Since != "" {
+		if since, err := tryParse(f.Since); err == nil {
+			ctx.Since = &since
+		} else {
+			if since, err := datemath.Parse(f.Since); err != nil {
+				return nil, fmt.Errorf("could not parse since: %s: %v", f.Since, err)
+			} else {
+				t := since.Time()
+				ctx.Since = &t
+			}
+		}
+		// add 1 second to the since time so that last_result.newest.modified can be used as a since
+		after := ctx.Since.Add(1 * time.Second)
+		ctx.Since = &after
+	}
 	return ctx, nil
 }
 
+var RFC3339NanoWithoutTimezone = "2006-01-02T15:04:05.999999999"
+
+func tryParse(s string) (time.Time, error) {
+	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+		return t, nil
+	}
+	if t, err := time.Parse(RFC3339NanoWithoutTimezone, s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, fmt.Errorf("could not parse %s", s)
+}
+
 func (f *FolderFilterContext) Filter(i fs.FileInfo) bool {
 	if i.IsDir() {
 		return false
@@ -120,6 +174,9 @@ func (f *FolderFilterContext) Filter(i fs.FileInfo) bool {
 	if f.regex != nil && !f.regex.MatchString(i.Name()) {
 		return false
 	}
+	if f.Since != nil && i.ModTime().Before(*f.Since) {
+		return false
+	}
 	return true
 }
 

checks/common.go

@@ -47,7 +47,7 @@ func getNextRuntime(canary v1.Canary, lastRuntime time.Time) (*time.Time, error)
 }
 
 // unstructure marshalls a struct to and from JSON to remove any type details
-func unstructure(o any) (out interface{}, err error) {
+func unstructure(o any) (out map[string]any, err error) {
 	data, err := json.Marshal(o)
 	if err != nil {
 		return nil, err
@@ -59,7 +59,13 @@ func unstructure(o any) (out interface{}, err error) {
 func template(ctx *context.Context, template v1.Template) (string, error) {
 	tpl := template.Gomplate()
 
-	ctx.InjectFunctions(&tpl)
+	if tpl.Functions == nil {
+		tpl.Functions = make(map[string]func() any)
+	}
+
+	for k, v := range ctx.GetContextualFunctions() {
+		tpl.Functions[k] = v
+	}
 
 	return gomplate.RunTemplate(ctx.Environment, tpl)
 }
@@ -137,7 +143,6 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e
 		if t.Start != nil {
 			in.Start = *t.Start
 		}
-
 		if t.Pass != nil {
 			in.Pass = *t.Pass
 		}
@@ -157,7 +162,12 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e
 			in.Error = t.Error
 		}
 		if t.Detail != nil {
-			in.Detail = t.Detail
+			if t.Detail == "$delete" {
+				in.Detail = nil
+				delete(in.Data, "results")
+			} else {
+				in.Detail = t.Detail
+			}
 		}
 		if t.DisplayType != "" {
 			in.DisplayType = t.DisplayType

checks/folder.go

@@ -1,6 +1,7 @@
 package checks
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -12,6 +13,8 @@ import (
 	"github.com/flanksource/canary-checker/pkg"
 )
 
+const SizeNotSupported = -1
+
 var (
 	bucketScanObjectCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -58,6 +61,15 @@ func (c *FolderChecker) Run(ctx *context.Context) pkg.Results {
 func (c *FolderChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
 	check := extConfig.(v1.FolderCheck)
 	path := strings.ToLower(check.Path)
+	ctx = ctx.WithCheck(check)
+	if ctx.CanTemplate() {
+		if err := ctx.TemplateStruct(&check.Filter); err != nil {
+			return pkg.Invalid(check, ctx.Canary, fmt.Sprintf("failed to template filter: %v", err))
+		}
+	}
+	if ctx.IsDebug() {
+		ctx.Infof("Checking %s with filter(%s)", path, check.Filter)
+	}
 	switch {
 	case strings.HasPrefix(path, "s3://"):
 		return CheckS3Bucket(ctx, check)
@@ -104,7 +116,11 @@ func getLocalFolderCheck(path string, filter v1.FolderFilter) (*FolderCheck, err
 		if err != nil {
 			return nil, err
 		}
-		return &FolderCheck{Oldest: info, Newest: info}, nil
+		return &FolderCheck{
+			Oldest:        newFile(info),
+			Newest:        newFile(info),
+			AvailableSize: SizeNotSupported,
+			TotalSize:     SizeNotSupported}, nil
 	}
 
 	for _, file := range files {
@@ -137,7 +153,11 @@ func getGenericFolderCheck(fs Filesystem, dir string, filter v1.FolderFilter) (*
 		if err != nil {
 			return nil, err
 		}
-		return &FolderCheck{Oldest: info, Newest: info}, nil
+		return &FolderCheck{
+			Oldest:        newFile(info),
+			Newest:        newFile(info),
+			AvailableSize: SizeNotSupported,
+			TotalSize:     SizeNotSupported}, nil
 	}
 
 	for _, file := range files {