sonatype nexus - quirks modes (#782)

## Description

Two step token auth doesn't work with Nexus and the REST endpoint for a
dedicated version doesn't exist. With this PR, Read-Access is done only
via Basic-Auth and in case the specified version endpoint is not found,
it will now try to load the complete package description from one level
above. That should contain ALL existing versions, incl. the requested
one.

## What type of PR is this? (check all applicable)

- [ ] 🍕 Feature
- [ ] 🎇 Restructuring
- [ ] 🐛 Bug Fix
- [ ] 📝 Documentation Update
- [ ] 🎨 Style
- [ ] 🧑‍💻 Code Refactor
- [ ] 🔥 Performance Improvements
- [ ] ✅ Test
- [ ] 🤖 Build
- [ ] 🔁 CI
- [ ] 📦 Chore (Release)
- [ ] ⏩ Revert

## Related Tickets & Documents

- Fixes #753 and #769

## Added tests?

- [ ] 👍 yes
- [ ] 🙅 no, because they aren't needed
- [ ] 🙋 no, because I need help
- [ ] Separate ticket for tests # (issue/pr)

Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration

## Added to documentation?

- [ ] 📜 README.md
- [ ] 🙅 no documentation needed

## Checklist:

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream
modules

docs/reference/ocm_credential-handling.md

@@ -167,7 +167,7 @@ The following credential consumer types are used/supported:
       - <code>password</code>: the basic auth password
 
 
-  - <code>NpmRegistry</code>: NPM repository
+  - <code>NpmRegistry</code>: NPM registry
 
     It matches the <code>NpmRegistry</code> consumer type and additionally acts like
     the <code>hostpath</code> type.

docs/reference/ocm_get_credentials.md

@@ -93,7 +93,7 @@ Matchers exist for the following usage contexts or consumer types:
       - <code>password</code>: the basic auth password
 
 
-  - <code>NpmRegistry</code>: NPM repository
+  - <code>NpmRegistry</code>: NPM registry
 
     It matches the <code>NpmRegistry</code> consumer type and additionally acts like
     the <code>hostpath</code> type.

pkg/contexts/credentials/builtin/npm/identity/identity.go

@@ -1,11 +1,8 @@
 package identity
 
 import (
-	"path"
+	"net/url"
 
-	. "net/url"
-
-	"github.com/open-component-model/ocm/pkg/common"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/cpi"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath"
 	"github.com/open-component-model/ocm/pkg/listformat"
@@ -37,31 +34,35 @@ func init() {
 		ATTR_TOKEN, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!",
 	})
 
-	cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository
+	cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM registry
 
 It matches the <code>`+CONSUMER_TYPE+`</code> consumer type and additionally acts like 
 the <code>`+hostpath.IDENTITY_TYPE+`</code> type.`,
 		attrs)
 }
 
-func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity {
-	url, err := Parse(rawURL)
+var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE)
+
+func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool {
+	return identityMatcher(pattern, cur, id)
+}
+
+func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) {
+	_url, err := url.JoinPath(rawURL, groupId)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-
-	url.Path = path.Join(url.Path, pkgName)
-	return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String())
+	return hostpath.GetConsumerIdentity(CONSUMER_TYPE, _url), nil
 }
 
-func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties {
-	id := GetConsumerId(repoUrl, pkgName)
-	if id == nil {
-		return nil
+func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) (cpi.Credentials, error) {
+	id, err := GetConsumerId(repoUrl, pkgName)
+	if err != nil {
+		return nil, err
 	}
-	credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id)
-	if credentials == nil || err != nil {
-		return nil
+	if id == nil {
+		logging.DynamicLogger(REALM).Debug("No consumer identity found.", "url", repoUrl, "groupId", pkgName)
+		return nil, nil
 	}
-	return credentials.Properties()
+	return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id)
 }

pkg/contexts/credentials/repositories/npm/config_test.go

@@ -3,31 +3,27 @@ package npm_test
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	. "github.com/open-component-model/ocm/pkg/testutils"
 
 	"github.com/open-component-model/ocm/pkg/common"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm"
+	. "github.com/open-component-model/ocm/pkg/testutils"
 )
 
 var _ = Describe("Config deserialization Test Environment", func() {
 	It("read .npmrc", func() {
 		ctx := credentials.New()
-
 		repo := Must(npm.NewRepository(ctx, "testdata/.npmrc"))
 		Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"}))
 		Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"}))
 	})
 
 	It("propagates credentials", func() {
 		ctx := credentials.New()
-
 		spec := npm.NewRepositorySpec("testdata/.npmrc")
-
 		_ = Must(ctx.RepositoryForSpec(spec))
-		id := identity.GetConsumerId("registry.npmjs.org", "pkg")
-
+		id := Must(identity.GetConsumerId("registry.npmjs.org", "pkg"))
 		creds := Must(credentials.CredentialsForConsumer(ctx, id))
 		Expect(creds).NotTo(BeNil())
 		Expect(creds.GetProperty(identity.ATTR_TOKEN)).To(Equal("npm_TOKEN"))
@@ -39,5 +35,4 @@ var _ = Describe("Config deserialization Test Environment", func() {
 		Expect(t).NotTo(BeNil())
 		Expect(t.Description()).NotTo(Equal(""))
 	})
-
 })

pkg/contexts/credentials/repositories/npm/provider.go

@@ -35,8 +35,12 @@ func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi.
 	var creds cpi.CredentialsSource
 
 	for key, value := range all {
-		id := npm.GetConsumerId("https://"+key, "")
-
+		id, err := npm.GetConsumerId("https://"+key, "")
+		if err != nil {
+			log := logging.Context().Logger(npm.REALM)
+			log.LogError(err, "Failed to get consumer id", "key", key, "value", value)
+			return nil, nil
+		}
 		if m(requested, currentFound, id) {
 			creds = newCredentials(value)
 			currentFound = id

pkg/contexts/credentials/repositories/npm/repository.go

@@ -65,7 +65,7 @@ func (r *Repository) Read(force bool) error {
 	}
 
 	if r.path == "" {
-		return fmt.Errorf("npmrc path not provided")
+		return errors.New("npmrc path not provided")
 	}
 	cfg, path, err := readNpmConfigFile(r.path)
 	if err != nil {

pkg/contexts/ocm/accessmethods/npm/method.go

@@ -86,7 +86,7 @@ func (a *AccessSpec) AccessMethod(c accspeccpi.ComponentVersionAccess) (accspecc
 }
 
 func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.ComponentVersionAccess) string {
-	meta, err := a.getPackageMeta(access.GetContext())
+	meta, err := a.GetPackageVersion(access.GetContext())
 	if err != nil {
 		return ""
 	}
@@ -96,43 +96,70 @@ func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.Comp
 	return ""
 }
 
-// PackageUrl returns the URL of the NPM package (Registry/Package/Version).
+// PackageUrl returns the URL of the NPM package (Registry/Package).
 func (a *AccessSpec) PackageUrl() string {
-	return a.Registry + path.Join("/", a.Package, a.Version)
+	return strings.TrimSuffix(a.Registry, "/") + path.Join("/", a.Package)
 }
 
-func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) {
+// PackageVersionUrl returns the URL of the NPM package-version (Registry/Package/Version).
+func (a *AccessSpec) PackageVersionUrl() string {
+	return strings.TrimSuffix(a.Registry, "/") + path.Join("/", a.Package, a.Version)
+}
+
+func (a *AccessSpec) GetPackageVersion(ctx accspeccpi.Context) (*npm.Version, error) {
 	r, err := reader(a, vfsattr.Get(ctx), ctx)
 	if err != nil {
 		return nil, err
 	}
-	buf := &bytes.Buffer{}
-	_, err = io.Copy(buf, io.LimitReader(r, 200000))
+	defer r.Close()
+	buf, err := io.ReadAll(r)
 	if err != nil {
-		return nil, errors.Wrapf(err, "cannot get version metadata for %s", a.PackageUrl())
+		return nil, errors.Wrapf(err, "cannot get version metadata for %s", a.PackageVersionUrl())
 	}
-
-	var metadata meta
-
-	err = json.Unmarshal(buf.Bytes(), &metadata)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", a.PackageUrl())
+	var version npm.Version
+	err = json.Unmarshal(buf, &version)
+	if err != nil || version.Dist.Tarball == "" {
+		// ugly fallback as workaround for https://github.com/sonatype/nexus-public/issues/224
+		var project npm.Project
+		err = json.Unmarshal(buf, &project) // parse the complete project
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", a.PackageVersionUrl())
+		}
+		v, ok := project.Version[a.Version] // and pick only the specified version
+		if !ok {
+			return nil, errors.Newf("version '%s' doesn't exist", a.Version)
+		}
+		version = v
 	}
-	return &metadata, nil
+	return &version, nil
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) {
 	factory := func() (blobaccess.BlobAccess, error) {
-		meta, err := a.getPackageMeta(c.GetContext())
+		meta, err := a.GetPackageVersion(c.GetContext())
 		if err != nil {
 			return nil, err
 		}
 
 		f := func() (io.ReadCloser, error) {
 			return reader(a, vfsattr.Get(c.GetContext()), c.GetContext(), meta.Dist.Tarball)
 		}
+		if meta.Dist.Integrity != "" {
+			tf := f
+			f = func() (io.ReadCloser, error) {
+				r, err := tf()
+				if err != nil {
+					return nil, err
+				}
+				digest, err := iotools.DecodeBase64ToHex(meta.Dist.Integrity)
+				if err != nil {
+					return nil, err
+				}
+				return iotools.VerifyingReaderWithHash(r, crypto.SHA512, digest), nil
+			}
+		}
 		if meta.Dist.Shasum != "" {
 			tf := f
 			f = func() (io.ReadCloser, error) {
@@ -149,15 +176,8 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A
 	return accspeccpi.NewDefaultMethodImpl(c, a, "", mime.MIME_TGZ, factory), nil
 }
 
-type meta struct {
-	Dist struct {
-		Shasum  string `json:"shasum"`
-		Tarball string `json:"tarball"`
-	} `json:"dist"`
-}
-
 func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...string) (io.ReadCloser, error) {
-	url := a.PackageUrl()
+	url := a.PackageVersionUrl()
 	if len(tar) > 0 {
 		url = tar[0]
 	}
@@ -170,12 +190,38 @@ func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...st
 	if err != nil {
 		return nil, err
 	}
-	npm.Authorize(req, ctx, a.Registry, a.Package)
+	err = npm.BasicAuth(req, ctx, a.Registry, a.Package)
+	if err != nil {
+		return nil, err
+	}
 	c := &http.Client{}
 	resp, err := c.Do(req)
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		// maybe it's stupid Nexus - https://github.com/sonatype/nexus-public/issues/224?
+		url = a.PackageUrl()
+		req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+		if err != nil {
+			return nil, err
+		}
+		err = npm.BasicAuth(req, ctx, a.Registry, a.Package)
+		if err != nil {
+			return nil, err
+		}
+
+		// close body before overwriting to close any pending connections
+		resp.Body.Close()
+		resp, err = c.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		defer resp.Body.Close()
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		defer resp.Body.Close()
 		buf := &bytes.Buffer{}
@@ -185,5 +231,9 @@ func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...st
 		}
 		return nil, errors.Newf("version meta data request %s provides %s: %s", url, resp.Status, buf.String())
 	}
-	return resp.Body, nil
+	content, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(bytes.NewBuffer(content)), nil
 }

pkg/contexts/ocm/accessmethods/npm/method_test.go

@@ -5,15 +5,15 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	. "github.com/open-component-model/ocm/pkg/env"
-	. "github.com/open-component-model/ocm/pkg/env/builder"
-	. "github.com/open-component-model/ocm/pkg/testutils"
 
 	"github.com/open-component-model/ocm/pkg/contexts/ocm"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
+	. "github.com/open-component-model/ocm/pkg/env"
+	. "github.com/open-component-model/ocm/pkg/env/builder"
 	"github.com/open-component-model/ocm/pkg/iotools"
 	"github.com/open-component-model/ocm/pkg/mime"
+	. "github.com/open-component-model/ocm/pkg/testutils"
 )
 
 const NPMPATH = "/testdata/registry"
@@ -62,4 +62,20 @@ var _ = Describe("Method", func() {
 		_, err := m.Reader()
 		Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found 34a77645201d1a8fc5213ace787c220eabbd0967")))
 	})
+
+	It("PackageUrl()", func() {
+		packageUrl := "https://registry.npmjs.org/yargs"
+		acc := npm.New("https://registry.npmjs.org", "yargs", "17.7.1")
+		Expect(acc.PackageUrl()).To(Equal(packageUrl))
+		acc = npm.New("https://registry.npmjs.org/", "yargs", "17.7.1")
+		Expect(acc.PackageUrl()).To(Equal(packageUrl))
+	})
+
+	It("PackageVersionUrl()", func() {
+		packageVersionUrl := "https://registry.npmjs.org/yargs/17.7.1"
+		acc := npm.New("https://registry.npmjs.org", "yargs", "17.7.1")
+		Expect(acc.PackageVersionUrl()).To(Equal(packageVersionUrl))
+		acc = npm.New("https://registry.npmjs.org/", "yargs", "17.7.1")
+		Expect(acc.PackageVersionUrl()).To(Equal(packageVersionUrl))
+	})
 })