From 8a597b83c5a7d47a8f79e7873aa7ba4b322567de Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Thu, 10 Oct 2024 15:32:40 +0200 Subject: [PATCH] CLI: fix recursive mode for ocm hash component (#954) #### What this PR does / why we need it In recursive mode of `ocm hash components`, the closure calculation is done in the processing chain after the mapping to the hash object. This cause type assertion panics. Additionally, the recalculating hashes for a component version using a shared state, also a panic occurs, because it has already been calculated as part of a closure. Both problems are fixed with this PR. Then, it adds some more flexibility to the input handling, enabling the use of the constructor processing in a more flexible way for other programs besides the usual cli commands. #### Which issue(s) this PR fixes --- api/ocm/internal/modopts.go | 3 ++- api/ocm/tools/signing/handle.go | 7 ++++++ .../ocmcmds/common/addhdlrs/interface.go | 15 ++++++++++++- .../commands/ocmcmds/common/addhdlrs/utils.go | 3 ++- .../ocmcmds/common/cmds/signing/cmd.go | 1 + cmds/ocm/commands/ocmcmds/common/resources.go | 3 ++- .../commands/ocmcmds/components/hash/cmd.go | 4 ++-- .../ocmcmds/components/hash/cmd_test.go | 22 +++++++++++++++++++ cmds/ocm/common/processing/processing.go | 9 ++++++++ 9 files changed, 61 insertions(+), 6 deletions(-) diff --git a/api/ocm/internal/modopts.go b/api/ocm/internal/modopts.go index 841b12d5e..41826374b 100644 --- a/api/ocm/internal/modopts.go +++ b/api/ocm/internal/modopts.go @@ -3,6 +3,7 @@ package internal import ( "fmt" + "github.com/mandelsoft/goutils/general" "github.com/mandelsoft/goutils/optionutils" "ocm.software/ocm/api/ocm/compdesc" @@ -212,7 +213,7 @@ func (m *ModificationOptions) ApplyModificationOption(opts *ModificationOptions) } func (m *ModificationOptions) GetHasher(algo ...string) Hasher { - return m.HasherProvider.GetHasher(utils.OptionalDefaulted(m.DefaultHashAlgorithm, algo...)) + return m.HasherProvider.GetHasher(general.OptionalDefaulted(m.DefaultHashAlgorithm, algo...)) } func NewModificationOptions(list ...ModificationOption) *ModificationOptions { diff --git a/api/ocm/tools/signing/handle.go b/api/ocm/tools/signing/handle.go index 0b5e8af61..0ef1a649c 100644 --- a/api/ocm/tools/signing/handle.go +++ b/api/ocm/tools/signing/handle.go @@ -156,6 +156,13 @@ func apply(state WalkingState, cv ocm.ComponentVersionAccess, opts *Options, clo return vi.digestingContexts[state.Context.CtxKey], err } } + if vi != nil { + // check for already calculated + dc := vi.GetContext(nv) + if dc != nil { + return dc, nil + } + } return _apply(state, nv, cv, vi, opts) } diff --git a/cmds/ocm/commands/ocmcmds/common/addhdlrs/interface.go b/cmds/ocm/commands/ocmcmds/common/addhdlrs/interface.go index 8a969e6d0..6faf055c2 100644 --- a/cmds/ocm/commands/ocmcmds/common/addhdlrs/interface.go +++ b/cmds/ocm/commands/ocmcmds/common/addhdlrs/interface.go @@ -17,11 +17,24 @@ import ( // a content based element (sources or resources). // It is either an input or access specification. type ResourceInput struct { - Access *cpi.GenericAccessSpec `json:"access"` + // SourceFile described the original source (file) the input + // is taken from. By default, this is not set since it is taken from the + // file information of the processed constructor resource. + // If an input aggregated in a constructor resource is provided + // by some other source, this field can be set. + // The source information is finally used by the input implementations + // to evaluate relative path specifications in the input specification. + // This should always relate to the original source. + SourceFile string `json:"sourceFile,omitempty"` + Access *cpi.GenericAccessSpec `json:"access"` // Input *inputs.BlobInput `json:"input,omitempty"` Input *inputs.GenericInputSpec `json:"input,omitempty"` } +func (r *ResourceInput) SetSourceFile(s string) { + r.SourceFile = s +} + // ElementSpecHandler is the interface for a handler // responsible to handle a dedicated kind of element specification. type ElementSpecHandler interface { diff --git a/cmds/ocm/commands/ocmcmds/common/addhdlrs/utils.go b/cmds/ocm/commands/ocmcmds/common/addhdlrs/utils.go index c47f272ee..1e8a02d0a 100644 --- a/cmds/ocm/commands/ocmcmds/common/addhdlrs/utils.go +++ b/cmds/ocm/commands/ocmcmds/common/addhdlrs/utils.go @@ -7,6 +7,7 @@ import ( "io" "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/general" "github.com/mandelsoft/vfs/pkg/vfs" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/util/validation/field" @@ -162,7 +163,7 @@ func DetermineElementForData(ctx clictx.Context, ictx inputs.Context, si SourceI if err != nil { return nil, err } - if err = Validate(input, ictx, si.Origin()); err != nil { + if err = Validate(input, ictx, general.OptionalDefaulted(si.Origin(), input.SourceFile)); err != nil { return nil, err } } diff --git a/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go b/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go index 966cf6bb6..347def81a 100644 --- a/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go +++ b/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go @@ -139,6 +139,7 @@ func NewAction(desc []string, ctx ocm.Context, p common.Printer, sopts *signing. func (a *action) Digest(o *comphdlr.Object) (*metav1.DigestSpec, *compdesc.ComponentDescriptor, error) { sopts := *a.sopts sopts.Resolver = resolvers.NewCompoundResolver(o.Repository, a.sopts.Resolver) + d, err := signing.Apply(a.printer, &a.state, o.ComponentVersion, &sopts) var cd *compdesc.ComponentDescriptor nv := common.VersionedElementKey(o.ComponentVersion) diff --git a/cmds/ocm/commands/ocmcmds/common/resources.go b/cmds/ocm/commands/ocmcmds/common/resources.go index 0517c7b48..63c1d63c4 100644 --- a/cmds/ocm/commands/ocmcmds/common/resources.go +++ b/cmds/ocm/commands/ocmcmds/common/resources.go @@ -7,6 +7,7 @@ import ( _ "ocm.software/ocm/cmds/ocm/commands/ocmcmds/common/inputs/types" "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/general" "github.com/mandelsoft/goutils/sliceutils" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/spf13/pflag" @@ -479,7 +480,7 @@ func ProcessElements(ictx inputs.Context, cv ocm.ComponentVersionAccess, elems [ info := inputs.InputResourceInfo{ ComponentVersion: common.VersionedElementKey(cv), ElementName: elem.Spec().GetName(), - InputFilePath: elem.Source().Origin(), + InputFilePath: general.OptionalDefaulted(elem.Source().Origin(), elem.Input().SourceFile), } blob, hint, berr := elem.Input().Input.GetBlob(ictx, info) if berr != nil { diff --git a/cmds/ocm/commands/ocmcmds/components/hash/cmd.go b/cmds/ocm/commands/ocmcmds/components/hash/cmd.go index 0469ae56e..eeb4c41b9 100644 --- a/cmds/ocm/commands/ocmcmds/components/hash/cmd.go +++ b/cmds/ocm/commands/ocmcmds/components/hash/cmd.go @@ -102,8 +102,8 @@ func TableOutput(opts *output.Options, h *action, mapping processing.MappingFunc def := &output.TableOutput{ Headers: output.Fields("COMPONENT", "VERSION", "HASH", wide), Options: opts, - Chain: comphdlr.Sort.Map(h.digester), - Mapping: mapping, + Chain: comphdlr.Sort, + Mapping: processing.MappingSequence(h.digester, mapping), } return closureoption.TableOutput(def, comphdlr.ClosureExplode) } diff --git a/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go b/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go index baa23fd4c..afb826dd5 100644 --- a/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go +++ b/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go @@ -19,6 +19,7 @@ const ( ARCH = "/tmp/ca" VERSION = "v1" COMP = "test.de/x" + COMP2 = "test.de/y" PROVIDER = "mandelsoft" ) @@ -103,4 +104,25 @@ HASH : 6e8e9eb0af1c4c0b9dcc4161168b3f0ad913bc85e4234688dd6d4b283fe4b95 NORMALIZED FORM: {"component":{"componentReferences":[],"name":"test.de/x","provider":{"name":"mandelsoft"},"resources":[{"digest":{"hashAlgorithm":"SHA-256","normalisationAlgorithm":"genericBlobDigest/v1","value":"810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50"},"name":"test","relation":"local","type":"plainText","version":"v1"}],"sources":[],"version":"v1"}} ---`)) }) + + It("hash component recursively", func() { + env.OCMCommonTransport(ARCH, accessio.FormatDirectory, func() { + env.ComponentVersion(COMP2, VERSION, func() { + env.Provider(PROVIDER) + }) + env.ComponentVersion(COMP, VERSION, func() { + env.Provider(PROVIDER) + env.Reference("ref", COMP2, VERSION) + }) + }) + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute("hash", "components", "-r", ARCH)).To(Succeed()) + Expect(buf.String()).To(StringEqualTrimmedWithContext(` +REFERENCEPATH COMPONENT VERSION HASH IDENTITY + test.de/x v1 b74cee6c6b8215f470efd0e3c49618bb98610fc80de36a2e121d0550650b9cdc +test.de/x:v1 test.de/y v1 e60c791a20091abcf8d35742a134b3a99ce811d874fd721870b28ea90ef5ad2a "name"="ref" + test.de/y v1 e60c791a20091abcf8d35742a134b3a99ce811d874fd721870b28ea90ef5ad2a +`)) + }) }) diff --git a/cmds/ocm/common/processing/processing.go b/cmds/ocm/common/processing/processing.go index 61fb794fe..20dca99d6 100644 --- a/cmds/ocm/common/processing/processing.go +++ b/cmds/ocm/common/processing/processing.go @@ -102,3 +102,12 @@ func NewAsyncProcessingSource(log logging.Context, f func() data.Iterable, pool }) return p } + +func MappingSequence(mapper ...MappingFunction) MappingFunction { + return func(e interface{}) interface{} { + for _, m := range mapper { + e = m(e) + } + return e + } +}