diff --git a/api/context/context.go b/api/context/context.go index 5532c12e5..abc32c1bc 100644 --- a/api/context/context.go +++ b/api/context/context.go @@ -84,26 +84,27 @@ func getDomain(username string) string { return "" } -func (ctx *Context) GetFunctionsFor(check external.Check) map[string]any { - env := make(map[string]any) - - return env -} - func (ctx *Context) Template(check external.Check, template string) (string, error) { env := ctx.Environment - for k, v := range ctx.GetFunctionsFor(check) { - env[k] = v + tpl := gomplate.Template{Template: template} + if tpl.Functions == nil { + tpl.Functions = make(map[string]func() any) } - - out, err := gomplate.RunExpression(env, gomplate.Template{Template: template}) + for k, v := range ctx.GetContextualFunctions() { + tpl.Functions[k] = v + } + out, err := gomplate.RunExpression(env, tpl) if err != nil { return "", err } return fmt.Sprintf("%s", out), nil } +func (ctx *Context) CanTemplate() bool { + return ctx.Canary.Annotations["template"] != "false" +} + func (ctx *Context) GetConnection(conn v1.Connection) (*models.Connection, error) { var _conn *models.Connection var err error @@ -158,6 +159,21 @@ func (ctx *Context) GetConnection(conn v1.Connection) (*models.Connection, error return _conn, nil } +func (ctx Context) TemplateStruct(o interface{}) error { + templater := gomplate.StructTemplater{ + Values: ctx.Environment, + Funcs: ctx.GetContextualFunctions(), + // access go values in template requires prefix everything with . + // to support $(username) instead of $(.username) we add a function for each var + ValueFunctions: true, + DelimSets: []gomplate.Delims{ + {Left: "{{", Right: "}}"}, + {Left: "$(", Right: ")"}, + }, + } + return templater.Walk(o) +} + func (ctx Context) GetAuthValues(auth v1.Authentication) (v1.Authentication, error) { // in case nil we are sending empty string values for username and password if auth.IsEmpty() { diff --git a/api/context/functions.go b/api/context/functions.go index 886f9391d..7623f9e30 100644 --- a/api/context/functions.go +++ b/api/context/functions.go @@ -5,16 +5,13 @@ import ( "time" "github.com/flanksource/commons/logger" - "github.com/flanksource/gomplate/v3" ) -func (ctx *Context) InjectFunctions(template *gomplate.Template) { +func (ctx *Context) GetContextualFunctions() map[string]func() any { + funcs := make(map[string]func() any) if check, ok := ctx.Environment["check"]; ok { checkID := check.(map[string]any)["id"] - if template.Functions == nil { - template.Functions = make(map[string]func() any) - } - template.Functions["last_result"] = func() any { + funcs["last_result"] = func() any { if ctx.cache == nil { ctx.cache = make(map[string]any) } @@ -78,4 +75,5 @@ func (ctx *Context) InjectFunctions(template *gomplate.Template) { return status } } + return funcs } diff --git a/api/v1/common.go b/api/v1/common.go index da6f17f12..1d6990f4e 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -1,6 +1,7 @@ package v1 import ( + "fmt" "io/fs" "net/url" "regexp" @@ -13,6 +14,7 @@ import ( "github.com/flanksource/commons/duration" "github.com/flanksource/duty/types" "github.com/flanksource/gomplate/v3" + "github.com/timberio/go-datemath" ) type Duration string @@ -49,16 +51,41 @@ func (s Size) Value() (*int64, error) { type FolderFilter struct { MinAge Duration `yaml:"minAge,omitempty" json:"minAge,omitempty"` MaxAge Duration `yaml:"maxAge,omitempty" json:"maxAge,omitempty"` + Since string `yaml:"since,omitempty" json:"since,omitempty"` MinSize Size `yaml:"minSize,omitempty" json:"minSize,omitempty"` MaxSize Size `yaml:"maxSize,omitempty" json:"maxSize,omitempty"` Regex string `yaml:"regex,omitempty" json:"regex,omitempty"` } +func (f FolderFilter) String() string { + s := []string{} + if f.MinAge != "" { + s = append(s, fmt.Sprintf("minAge="+string(f.MinAge))) + } + if f.MaxAge != "" { + s = append(s, "maxAge="+string(f.MaxAge)) + } + if f.MinSize != "" { + s = append(s, "minSize="+string(f.MinSize)) + } + if f.MaxSize != "" { + s = append(s, "maxSize="+string(f.MaxSize)) + } + if f.Regex != "" { + s = append(s, "regex="+f.Regex) + } + if f.Since != "" { + s = append(s, "since="+f.Since) + } + return strings.Join(s, ", ") +} + // +k8s:deepcopy-gen=false type FolderFilterContext struct { FolderFilter minAge, maxAge *time.Duration minSize, maxSize *int64 + Since *time.Time // kubebuilder:object:generate=false regex *regexp.Regexp } @@ -98,9 +125,36 @@ func (f FolderFilter) New() (*FolderFilterContext, error) { return nil, err } } + if f.Since != "" { + if since, err := tryParse(f.Since); err == nil { + ctx.Since = &since + } else { + if since, err := datemath.Parse(f.Since); err != nil { + return nil, fmt.Errorf("could not parse since: %s: %v", f.Since, err) + } else { + t := since.Time() + ctx.Since = &t + } + } + // add 1 second to the since time so that last_result.newest.modified can be used as a since + after := ctx.Since.Add(1 * time.Second) + ctx.Since = &after + } return ctx, nil } +var RFC3339NanoWithoutTimezone = "2006-01-02T15:04:05.999999999" + +func tryParse(s string) (time.Time, error) { + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + if t, err := time.Parse(RFC3339NanoWithoutTimezone, s); err == nil { + return t, nil + } + return time.Time{}, fmt.Errorf("could not parse %s", s) +} + func (f *FolderFilterContext) Filter(i fs.FileInfo) bool { if i.IsDir() { return false @@ -120,6 +174,9 @@ func (f *FolderFilterContext) Filter(i fs.FileInfo) bool { if f.regex != nil && !f.regex.MatchString(i.Name()) { return false } + if f.Since != nil && i.ModTime().Before(*f.Since) { + return false + } return true } diff --git a/checks/common.go b/checks/common.go index 50957d529..743268ff0 100644 --- a/checks/common.go +++ b/checks/common.go @@ -47,7 +47,7 @@ func getNextRuntime(canary v1.Canary, lastRuntime time.Time) (*time.Time, error) } // unstructure marshalls a struct to and from JSON to remove any type details -func unstructure(o any) (out interface{}, err error) { +func unstructure(o any) (out map[string]any, err error) { data, err := json.Marshal(o) if err != nil { return nil, err @@ -59,7 +59,13 @@ func unstructure(o any) (out interface{}, err error) { func template(ctx *context.Context, template v1.Template) (string, error) { tpl := template.Gomplate() - ctx.InjectFunctions(&tpl) + if tpl.Functions == nil { + tpl.Functions = make(map[string]func() any) + } + + for k, v := range ctx.GetContextualFunctions() { + tpl.Functions[k] = v + } return gomplate.RunTemplate(ctx.Environment, tpl) } @@ -137,7 +143,6 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e if t.Start != nil { in.Start = *t.Start } - if t.Pass != nil { in.Pass = *t.Pass } @@ -157,7 +162,12 @@ func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, e in.Error = t.Error } if t.Detail != nil { - in.Detail = t.Detail + if t.Detail == "$delete" { + in.Detail = nil + delete(in.Data, "results") + } else { + in.Detail = t.Detail + } } if t.DisplayType != "" { in.DisplayType = t.DisplayType diff --git a/checks/folder.go b/checks/folder.go index b0246e978..b19759f1c 100644 --- a/checks/folder.go +++ b/checks/folder.go @@ -1,6 +1,7 @@ package checks import ( + "fmt" "os" "strings" @@ -12,6 +13,8 @@ import ( "github.com/flanksource/canary-checker/pkg" ) +const SizeNotSupported = -1 + var ( bucketScanObjectCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -58,6 +61,15 @@ func (c *FolderChecker) Run(ctx *context.Context) pkg.Results { func (c *FolderChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results { check := extConfig.(v1.FolderCheck) path := strings.ToLower(check.Path) + ctx = ctx.WithCheck(check) + if ctx.CanTemplate() { + if err := ctx.TemplateStruct(&check.Filter); err != nil { + return pkg.Invalid(check, ctx.Canary, fmt.Sprintf("failed to template filter: %v", err)) + } + } + if ctx.IsDebug() { + ctx.Infof("Checking %s with filter(%s)", path, check.Filter) + } switch { case strings.HasPrefix(path, "s3://"): return CheckS3Bucket(ctx, check) @@ -104,7 +116,11 @@ func getLocalFolderCheck(path string, filter v1.FolderFilter) (*FolderCheck, err if err != nil { return nil, err } - return &FolderCheck{Oldest: info, Newest: info}, nil + return &FolderCheck{ + Oldest: newFile(info), + Newest: newFile(info), + AvailableSize: SizeNotSupported, + TotalSize: SizeNotSupported}, nil } for _, file := range files { @@ -137,7 +153,11 @@ func getGenericFolderCheck(fs Filesystem, dir string, filter v1.FolderFilter) (* if err != nil { return nil, err } - return &FolderCheck{Oldest: info, Newest: info}, nil + return &FolderCheck{ + Oldest: newFile(info), + Newest: newFile(info), + AvailableSize: SizeNotSupported, + TotalSize: SizeNotSupported}, nil } for _, file := range files { diff --git a/checks/folder_check.go b/checks/folder_check.go index f79c882e9..4b3834ec8 100644 --- a/checks/folder_check.go +++ b/checks/folder_check.go @@ -15,31 +15,47 @@ type Filesystem interface { } type FolderCheck struct { - Oldest os.FileInfo - Newest os.FileInfo - MinSize os.FileInfo - MaxSize os.FileInfo - SupportsTotalSize bool - SupportsAvailableSize bool - TotalSize int64 - AvailableSize int64 - Files []os.FileInfo + Oldest *File `json:"oldest,omitempty"` + Newest *File `json:"newest,omitempty"` + MinSize *File `json:"smallest,omitempty"` + MaxSize *File `json:"largest,omitempty"` + TotalSize int64 `json:"size,omitempty"` + AvailableSize int64 `json:"availableSize,omitempty"` + Files []File `json:"files,omitempty"` } -func (f *FolderCheck) Append(file os.FileInfo) { - if f.Oldest == nil || f.Oldest.ModTime().After(file.ModTime()) { +type File struct { + Name string `json:"name,omitempty"` + Size int64 `json:"size,omitempty"` + Mode string `json:"mode,omitempty"` + Modified time.Time `json:"modified"` + IsDir bool `json:"is_dir,omitempty"` +} + +func newFile(file os.FileInfo) *File { + return &File{ + Name: file.Name(), + Size: file.Size(), + Mode: file.Mode().String(), + Modified: file.ModTime().UTC(), + IsDir: file.IsDir(), + } +} +func (f *FolderCheck) Append(osFile os.FileInfo) { + file := newFile(osFile) + if f.Oldest == nil || f.Oldest.Modified.After(osFile.ModTime()) { f.Oldest = file } - if f.Newest == nil || f.Newest.ModTime().Before(file.ModTime()) { + if f.Newest == nil || f.Newest.Modified.Before(osFile.ModTime()) { f.Newest = file } - if f.MinSize == nil || f.MinSize.Size() > file.Size() { + if f.MinSize == nil || f.MinSize.Size > osFile.Size() { f.MinSize = file } - if f.MaxSize == nil || f.MaxSize.Size() < file.Size() { + if f.MaxSize == nil || f.MaxSize.Size < osFile.Size() { f.MaxSize = file } - f.Files = append(f.Files, file) + f.Files = append(f.Files, *file) } func (f FolderCheck) Test(test v1.FolderTest) string { @@ -60,7 +76,7 @@ func (f FolderCheck) Test(test v1.FolderTest) string { } if test.AvailableSize != "" { - if !f.SupportsAvailableSize { + if f.AvailableSize == SizeNotSupported { return "available size not supported" } size, err := test.AvailableSize.Value() @@ -73,7 +89,7 @@ func (f FolderCheck) Test(test v1.FolderTest) string { } if test.TotalSize != "" { - if !f.SupportsTotalSize { + if f.TotalSize == SizeNotSupported { return "total size not supported" } size, err := test.TotalSize.Value() @@ -84,15 +100,16 @@ func (f FolderCheck) Test(test v1.FolderTest) string { return fmt.Sprintf("total size too small: %v < %v", mb(f.TotalSize), test.TotalSize) } } + if len(f.Files) == 0 { // nothing run age/size checks on return "" } - if minAge != nil && time.Since(f.Newest.ModTime()) < *minAge { - return fmt.Sprintf("%s is too new: %s < %s", f.Newest.Name(), age(f.Newest.ModTime()), test.MinAge) + if minAge != nil && time.Since(f.Newest.Modified) < *minAge { + return fmt.Sprintf("%s is too new: %s < %s", f.Newest.Name, age(f.Newest.Modified), test.MinAge) } - if maxAge != nil && time.Since(f.Oldest.ModTime()) > *maxAge { - return fmt.Sprintf("%s is too old %s > %s", f.Oldest.Name(), age(f.Oldest.ModTime()), test.MaxAge) + if maxAge != nil && time.Since(f.Oldest.Modified) > *maxAge { + return fmt.Sprintf("%s is too old %s > %s", f.Oldest.Name, age(f.Oldest.Modified), test.MaxAge) } if test.MinSize != "" { @@ -100,8 +117,8 @@ func (f FolderCheck) Test(test v1.FolderTest) string { if err != nil { return fmt.Sprintf("%s is an invalid size: %s", test.MinSize, err) } - if f.MinSize.Size() < *size { - return fmt.Sprintf("%s is too small: %v < %v", f.MinSize.Name(), mb(f.MinSize.Size()), test.MinSize) + if f.MinSize.Size < *size { + return fmt.Sprintf("%s is too small: %v < %v", f.MinSize.Name, mb(f.MinSize.Size), test.MinSize) } } @@ -110,8 +127,8 @@ func (f FolderCheck) Test(test v1.FolderTest) string { if err != nil { return fmt.Sprintf("%s is an invalid size: %s", test.MinSize, err) } - if f.MaxSize.Size() < *size { - return fmt.Sprintf("%s is too large: %v > %v", f.MaxSize.Name(), mb(f.MaxSize.Size()), test.MaxSize) + if f.MaxSize.Size < *size { + return fmt.Sprintf("%s is too large: %v > %v", f.MaxSize.Name, mb(f.MaxSize.Size), test.MaxSize) } } return "" diff --git a/checks/folder_smb.go b/checks/folder_smb.go index 92c23ebca..164fe6057 100644 --- a/checks/folder_smb.go +++ b/checks/folder_smb.go @@ -104,9 +104,6 @@ func CheckSmb(ctx *context.Context, check v1.FolderCheck) pkg.Results { return results.ErrorMessage(err) } - folders.SupportsAvailableSize = true - folders.SupportsTotalSize = true - folders.AvailableSize = int64(freeBlockCount * blockSize) folders.TotalSize = int64(totalBlockCount * blockSize) diff --git a/checks/folder_test.go b/checks/folder_test.go new file mode 100644 index 000000000..6a2a76a3b --- /dev/null +++ b/checks/folder_test.go @@ -0,0 +1,28 @@ +package checks + +import ( + "testing" + "time" + + v1 "github.com/flanksource/canary-checker/api/v1" + . "github.com/onsi/gomega" +) + +func TestFolderFilterSinceMath(t *testing.T) { + RegisterTestingT(t) + ctx, err := v1.FolderFilter{ + Since: "now-1h", + }.New() + + Expect(err).ToNot(HaveOccurred()) + Expect(*ctx.Since).To(BeTemporally("~", time.Now().Add(-1*time.Hour), 1*time.Second)) +} + +func TestFolderFilterSinceParse(t *testing.T) { + RegisterTestingT(t) + _, err := v1.FolderFilter{ + Since: "2023-10-31T19:18:57.14974Z", + }.New() + + Expect(err).ToNot(HaveOccurred()) +} diff --git a/checks/sql.go b/checks/sql.go index 0305ecab2..dfa4c648a 100644 --- a/checks/sql.go +++ b/checks/sql.go @@ -97,7 +97,7 @@ func CheckSQL(ctx *context.Context, checker SQLChecker) pkg.Results { // nolint: query := check.GetQuery() - if ctx.Canary.Annotations["template"] != "false" { + if ctx.CanTemplate() { query, err = template(ctx.WithCheck(checker.GetCheck()), v1.Template{ Template: query, }) diff --git a/cmd/operator.go b/cmd/operator.go index 3c47638b8..5c59b8386 100644 --- a/cmd/operator.go +++ b/cmd/operator.go @@ -59,8 +59,6 @@ func run(cmd *cobra.Command, args []string) { logger.Fatalf("failed to get zap logger") return } - canaryJobs.LogFail = logFail - canaryJobs.LogPass = logPass loggr := ctrlzap.NewRaw( ctrlzap.UseDevMode(true), diff --git a/cmd/root.go b/cmd/root.go index a4724d7c3..68769192b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,6 +19,10 @@ import ( var Root = &cobra.Command{ Use: "canary-checker", PersistentPreRun: func(cmd *cobra.Command, args []string) { + + canary.LogFail = logFail + canary.LogPass = logPass + logger.UseZap(cmd.Flags()) for _, script := range sharedLibrary { if err := gomplate.LoadSharedLibrary(script); err != nil { diff --git a/cmd/run.go b/cmd/run.go index 4c2c525b9..ed550e724 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -114,12 +114,10 @@ var Run = &cobra.Command{ } } if jsonExport { - for _, result := range results { result.Name = def(result.Name, result.Check.GetName(), result.Canary.Name) result.Description = def(result.Description, result.Check.GetDescription()) result.Labels = merge(result.Check.GetLabels(), result.Labels) - fmt.Println(result.Name) } data, err := json.Marshal(results) diff --git a/cmd/serve.go b/cmd/serve.go index daa25c99e..6ef5c7ab5 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -45,6 +45,7 @@ var Serve = &cobra.Command{ Use: "serve config.yaml", Short: "Start a server to execute checks", Run: func(cmd *cobra.Command, configFiles []string) { + logger.ParseFlags(cmd.Flags()) setup() canaryJobs.StartScanCanaryConfigs(dataFile, configFiles) if executor { diff --git a/cmd/topology.go b/cmd/topology.go index 76e264fac..bf10373a5 100644 --- a/cmd/topology.go +++ b/cmd/topology.go @@ -25,6 +25,7 @@ var topologyRunNamespace string var Topology = &cobra.Command{ Use: "topology", PersistentPreRun: func(cmd *cobra.Command, args []string) { + logger.ParseFlags(cmd.Flags()) db.ConnectionString = readFromEnv(db.ConnectionString) if db.IsConfigured() { if err := db.Init(); err != nil { diff --git a/config/canary-dev.yaml b/config/canary-dev.yaml new file mode 100644 index 000000000..5ef40db53 --- /dev/null +++ b/config/canary-dev.yaml @@ -0,0 +1,119 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + deployment.kubernetes.io/revision: "4" + meta.helm.sh/release-name: canary-checker + meta.helm.sh/release-namespace: canary-checker + labels: + app.kubernetes.io/instance: canary-checker + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: canary-checker + app.kubernetes.io/version: master + control-plane: canary-checker + helm.sh/chart: canary-checker-0.0.0 + name: canary-checker + namespace: canary-checker +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/instance: canary-checker + app.kubernetes.io/name: canary-checker + control-plane: canary-checker + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/instance: canary-checker + app.kubernetes.io/name: canary-checker + control-plane: canary-checker + spec: + containers: + - args: + - operator + - -v + - --httpPort + - "8080" + - --disable-postgrest=false + - --db-migrations=true + - --cache-timeout=90 + - --default-window=1h + - --check-status-retention-period=180 + - --check-retention-period=7 + - --canary-retention-period=7 + command: + - /app/canary-checker + env: + - name: PING_MODE + value: privileged + - name: DB_URL + value: embedded:///opt/database/ + image: docker.io/flanksource/canary-checker-full:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: canary-checker + readinessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 2Gi + requests: + cpu: 200m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - CAP_NET_RAW + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/podinfo + name: podinfo + - mountPath: /app/canary-checker.properties + name: config + subPath: canary-checker.properties + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 1000 + serviceAccount: canary-checker-sa + serviceAccountName: canary-checker-sa + terminationGracePeriodSeconds: 30 + volumes: + - downwardAPI: + defaultMode: 420 + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: labels + name: podinfo + - configMap: + defaultMode: 420 + name: canary-checker + name: config diff --git a/config/deploy/crd.yaml b/config/deploy/crd.yaml index f5fcf8de3..626843271 100644 --- a/config/deploy/crd.yaml +++ b/config/deploy/crd.yaml @@ -2654,6 +2654,8 @@ spec: type: string regex: type: string + since: + type: string type: object gcpConnection: properties: diff --git a/config/deploy/manifests.yaml b/config/deploy/manifests.yaml index 4e94a3f73..a9ac1413f 100644 --- a/config/deploy/manifests.yaml +++ b/config/deploy/manifests.yaml @@ -2654,6 +2654,8 @@ spec: type: string regex: type: string + since: + type: string type: object gcpConnection: properties: diff --git a/config/schemas/canary.schema.json b/config/schemas/canary.schema.json index 83c440dfa..88aaacbcf 100644 --- a/config/schemas/canary.schema.json +++ b/config/schemas/canary.schema.json @@ -1525,6 +1525,9 @@ "maxAge": { "type": "string" }, + "since": { + "type": "string" + }, "minSize": { "type": "string" }, diff --git a/config/schemas/component.schema.json b/config/schemas/component.schema.json index 63dbc48a7..d9a2b74f6 100644 --- a/config/schemas/component.schema.json +++ b/config/schemas/component.schema.json @@ -1704,6 +1704,9 @@ "maxAge": { "type": "string" }, + "since": { + "type": "string" + }, "minSize": { "type": "string" }, diff --git a/config/schemas/health_folder.schema.json b/config/schemas/health_folder.schema.json index a308eacaa..3dedf2474 100644 --- a/config/schemas/health_folder.schema.json +++ b/config/schemas/health_folder.schema.json @@ -168,6 +168,9 @@ "maxAge": { "type": "string" }, + "since": { + "type": "string" + }, "minSize": { "type": "string" }, diff --git a/config/schemas/topology.schema.json b/config/schemas/topology.schema.json index c1d9b6efb..142d68f32 100644 --- a/config/schemas/topology.schema.json +++ b/config/schemas/topology.schema.json @@ -1674,6 +1674,9 @@ "maxAge": { "type": "string" }, + "since": { + "type": "string" + }, "minSize": { "type": "string" }, diff --git a/fixtures/datasources/folder_pass.yaml b/fixtures/datasources/folder_pass.yaml index 00b10e1d8..5e7ef73b8 100644 --- a/fixtures/datasources/folder_pass.yaml +++ b/fixtures/datasources/folder_pass.yaml @@ -3,8 +3,32 @@ kind: Canary metadata: name: folder-pass spec: - interval: 30 + interval: 10 folder: - path: /etc/ - name: min count pass - minCount: 10 \ No newline at end of file + name: Check for updated /etc files + filter: + # use the last known max, or 60 days ago if no last known max + since: | + {{- if last_result.results.max }} + {{ last_result.results.max }} + {{- else}} + now-60d + {{- end}} + transform: + # Save the newest modified time to the results, overriding the full file listing that would normally be saved + # if no new files detected, use the last known max + expr: | + { + "detail": { + "max": string(results.?newest.modified.orValue(last_result().results.?max.orValue("now-60d"))), + } + }.toJSON() + display: + expr: results.?files.orValue([]).map(i, i.name).join(", ") + test: + expr: results.?files.orValue([]).size() > 0 + metrics: + - name: new_files + value: results.?files.orValue([]).size() + type: counter diff --git a/go.mod b/go.mod index 10bf7b5e2..b06442e1f 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/fergusstrange/embedded-postgres v1.24.0 github.com/flanksource/commons v1.17.1 github.com/flanksource/duty v1.0.205 - github.com/flanksource/gomplate/v3 v3.20.22 + github.com/flanksource/gomplate/v3 v3.20.24 github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 github.com/flanksource/kommons v0.31.4 github.com/flanksource/postq v1.0.0 @@ -62,6 +62,7 @@ require ( github.com/sevennt/echo-pprof v0.1.1-0.20220616082843-66a461746b5f github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/timberio/go-datemath v0.1.0 go.mongodb.org/mongo-driver v1.12.1 go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 @@ -126,7 +127,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.7.0 // indirect - github.com/flanksource/mapstructure v1.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect @@ -207,6 +207,7 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ohler55/ojg v1.20.2 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect diff --git a/go.sum b/go.sum index db99acf13..f8d0269f4 100644 --- a/go.sum +++ b/go.sum @@ -820,15 +820,13 @@ github.com/flanksource/commons v1.17.1/go.mod h1:RDdQI0/QYC4GzicbDaXIvBPjWuQWKLz github.com/flanksource/duty v1.0.205 h1:sQq+J4TMx69NnoM4XxBcJZ8P5HM5GjY/7zcuv/IQGTo= github.com/flanksource/duty v1.0.205/go.mod h1:V3fgZdrBgN47lloIz7MedwD/tq4ycHI8zFOApzUpFv4= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= -github.com/flanksource/gomplate/v3 v3.20.22 h1:DOytkLh1aND8KruydfCnu9K9oRKPeoJj2qgFqQkGrpE= -github.com/flanksource/gomplate/v3 v3.20.22/go.mod h1:2GgHZ2vWmtDspJMBfUIryOuzJSwc8jU7Kw9fDLr0TMA= +github.com/flanksource/gomplate/v3 v3.20.24 h1:Hp77K3FSoOX5HtrO+rMi52nH0s3d/Sj5UV5+RxApLos= +github.com/flanksource/gomplate/v3 v3.20.24/go.mod h1:GKmptFMdr2LbOuqwQZrmo9a/UygyZ0pbXffks8MuYhE= github.com/flanksource/is-healthy v0.0.0-20230705092916-3b4cf510c5fc/go.mod h1:4pQhmF+TnVqJroQKY8wSnSp+T18oLson6YQ2M0qPHfQ= github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 h1:s6jf6P1pRfdvksVFjIXFRfnimvEYUR0/Mmla1EIjiRM= github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7/go.mod h1:BH5gh9JyEAuuWVP6Q5y9h43VozS0RfKyjNpM9L4v4hw= github.com/flanksource/kommons v0.31.4 h1:zksAgYjZuwPgS8XTejDIWEYB0nPSU1i3Jxcavm/vovI= github.com/flanksource/kommons v0.31.4/go.mod h1:70BPMzjTvejsqRyVyAm/ZCeZ176toCvauaZjU03svnE= -github.com/flanksource/mapstructure v1.6.0 h1:+1kJ+QsO1SxjAgktfLlpZXetsVSJ0uCLhGKrA4BtwTE= -github.com/flanksource/mapstructure v1.6.0/go.mod h1:dttg5+FFE2sp4D/CrcPCVqufNDrBggDaM+08nk5S8Ps= github.com/flanksource/postq v1.0.0 h1:qdnWRmE/18dM5UlZGI9fJOx7BIJEQXKHIOuKzjOqNp8= github.com/flanksource/postq v1.0.0/go.mod h1:nE3mLh0vpF43TT+0HszC0QmORB37xSWPVHmhOY6PqBk= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -1299,6 +1297,8 @@ github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/ohler55/ojg v1.20.2 h1:ragZXnawcsq/knqIHaflvIX/8RzzAKMZSluAZAz+RFs= +github.com/ohler55/ojg v1.20.2/go.mod h1:uHcD1ErbErC27Zhb5Df2jUjbseLLcmOCo6oxSr3jZxo= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1461,6 +1461,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/timberio/go-datemath v0.1.0 h1:1OUCvSIX1qXLJ57h12OWfgt6MNpJnsdNvrp8dLIUFtg= +github.com/timberio/go-datemath v0.1.0/go.mod h1:m7kjsbCuO4QKP3KLfnxiUZWiOiFXmxj30HeexjL3lc0= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index b0093a682..b5582abe5 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -13,7 +13,7 @@ require ( cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/iam v1.1.3 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/flanksource/gomplate/v3 v3.20.22 // indirect + github.com/flanksource/gomplate/v3 v3.20.24 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/gosimple/unidecode v1.0.1 // indirect @@ -43,7 +43,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/flanksource/duty v1.0.205 // indirect github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 // indirect - github.com/flanksource/mapstructure v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/inflect v0.19.0 // indirect @@ -90,6 +89,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/ohler55/ojg v1.20.2 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robertkrimen/otto v0.2.1 // indirect @@ -99,6 +99,7 @@ require ( github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + github.com/timberio/go-datemath v0.1.0 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index aec222f59..580fba0ef 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -706,13 +706,11 @@ github.com/flanksource/commons v1.17.1/go.mod h1:RDdQI0/QYC4GzicbDaXIvBPjWuQWKLz github.com/flanksource/duty v1.0.205 h1:sQq+J4TMx69NnoM4XxBcJZ8P5HM5GjY/7zcuv/IQGTo= github.com/flanksource/duty v1.0.205/go.mod h1:V3fgZdrBgN47lloIz7MedwD/tq4ycHI8zFOApzUpFv4= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= -github.com/flanksource/gomplate/v3 v3.20.22 h1:DOytkLh1aND8KruydfCnu9K9oRKPeoJj2qgFqQkGrpE= -github.com/flanksource/gomplate/v3 v3.20.22/go.mod h1:2GgHZ2vWmtDspJMBfUIryOuzJSwc8jU7Kw9fDLr0TMA= +github.com/flanksource/gomplate/v3 v3.20.24 h1:Hp77K3FSoOX5HtrO+rMi52nH0s3d/Sj5UV5+RxApLos= +github.com/flanksource/gomplate/v3 v3.20.24/go.mod h1:GKmptFMdr2LbOuqwQZrmo9a/UygyZ0pbXffks8MuYhE= github.com/flanksource/is-healthy v0.0.0-20230705092916-3b4cf510c5fc/go.mod h1:4pQhmF+TnVqJroQKY8wSnSp+T18oLson6YQ2M0qPHfQ= github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 h1:s6jf6P1pRfdvksVFjIXFRfnimvEYUR0/Mmla1EIjiRM= github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7/go.mod h1:BH5gh9JyEAuuWVP6Q5y9h43VozS0RfKyjNpM9L4v4hw= -github.com/flanksource/mapstructure v1.6.0 h1:+1kJ+QsO1SxjAgktfLlpZXetsVSJ0uCLhGKrA4BtwTE= -github.com/flanksource/mapstructure v1.6.0/go.mod h1:dttg5+FFE2sp4D/CrcPCVqufNDrBggDaM+08nk5S8Ps= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -1011,6 +1009,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/ohler55/ojg v1.20.2 h1:ragZXnawcsq/knqIHaflvIX/8RzzAKMZSluAZAz+RFs= +github.com/ohler55/ojg v1.20.2/go.mod h1:uHcD1ErbErC27Zhb5Df2jUjbseLLcmOCo6oxSr3jZxo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -1100,6 +1100,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/timberio/go-datemath v0.1.0 h1:1OUCvSIX1qXLJ57h12OWfgt6MNpJnsdNvrp8dLIUFtg= +github.com/timberio/go-datemath v0.1.0/go.mod h1:m7kjsbCuO4QKP3KLfnxiUZWiOiFXmxj30HeexjL3lc0= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/pkg/results.go b/pkg/results.go index 9b98d9f4c..d1d34f48e 100644 --- a/pkg/results.go +++ b/pkg/results.go @@ -12,8 +12,10 @@ type Results []*CheckResult func Fail(check external.Check, canary v1.Canary) *CheckResult { return &CheckResult{ - Check: check, - Data: make(map[string]interface{}), + Check: check, + Data: map[string]interface{}{ + "results": make(map[string]interface{}), + }, Start: time.Now(), Pass: false, Canary: canary, @@ -29,18 +31,35 @@ func SetupError(canary v1.Canary, err error) Results { Invalid: true, Error: err.Error(), Check: check, - Data: make(map[string]interface{}), + Data: map[string]interface{}{ + "results": make(map[string]interface{}), + }, }) } return results } +func Invalid(check external.Check, canary v1.Canary, reason string) Results { + return Results{&CheckResult{ + Start: time.Now(), + Pass: false, + Error: reason, + Check: check, + Data: map[string]interface{}{ + "results": make(map[string]interface{}), + }, + Canary: canary, + }} +} + func Success(check external.Check, canary v1.Canary) *CheckResult { return &CheckResult{ - Start: time.Now(), - Pass: true, - Check: check, - Data: make(map[string]interface{}), + Start: time.Now(), + Pass: true, + Check: check, + Data: map[string]interface{}{ + "results": make(map[string]interface{}), + }, Canary: canary, } }