Merge remote-tracking branch 'upstream/dev'

artifactory/services/utils/specutils.go

@@ -18,8 +18,14 @@ type Aql struct {
 	ItemsFind string `json:"items.find"`
 }
 
+type PathMapping struct {
+	Input  string `json:"input"`
+	Output string `json:"output"`
+}
+
 type CommonParams struct {
 	Aql              Aql
+	PathMapping      PathMapping
 	Pattern          string
 	Exclusions       []string
 	Target           string

distribution/services/utils/distributionutils.go

@@ -34,17 +34,23 @@ func NewReleaseBundleParams(name, version string) ReleaseBundleParams {
 
 func CreateBundleBody(releaseBundleParams ReleaseBundleParams, dryRun bool) (*ReleaseBundleBody, error) {
 	var bundleQueries []BundleQuery
+	var pathMappings []rtUtils.PathMapping
+
 	// Create release bundle queries
 	for _, specFile := range releaseBundleParams.SpecFiles {
+		// Create path mapping
+		if specFile.GetSpecType() == rtUtils.AQL {
+			pathMappings = distribution.CreatePathMappings(specFile.PathMapping.Input, specFile.PathMapping.Output)
+		} else {
+			pathMappings = distribution.CreatePathMappingsFromPatternAndTarget(specFile.Pattern, specFile.Target)
+		}
+
 		// Create AQL
 		aql, err := createAql(specFile)
 		if err != nil {
 			return nil, err
 		}
 
-		// Create path mapping
-		pathMappings := distribution.CreatePathMappings(specFile.Pattern, specFile.Target)
-
 		// Create added properties
 		addedProps := createAddedProps(specFile)
 

distribution/services/utils/releasebundlebody.go

@@ -1,6 +1,8 @@
 package utils
 
-import "github.com/jfrog/jfrog-client-go/utils/distribution"
+import (
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
+)
 
 // REST body for create and update a release bundle
 type ReleaseBundleBody struct {
@@ -22,10 +24,10 @@ type BundleSpec struct {
 }
 
 type BundleQuery struct {
-	QueryName    string                     `json:"query_name,omitempty"`
-	Aql          string                     `json:"aql,omitempty"`
-	PathMappings []distribution.PathMapping `json:"mappings,omitempty"`
-	AddedProps   []AddedProps               `json:"added_props,omitempty"`
+	QueryName    string              `json:"query_name,omitempty"`
+	Aql          string              `json:"aql,omitempty"`
+	PathMappings []utils.PathMapping `json:"mappings,omitempty"`
+	AddedProps   []AddedProps        `json:"added_props,omitempty"`
 }
 
 type AddedProps struct {

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/go-git/go-git/v5 v5.9.0
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/gookit/color v1.5.4
-	github.com/jfrog/build-info-go v1.9.15
-	github.com/jfrog/gofrog v1.3.1
+	github.com/jfrog/build-info-go v1.9.16
+	github.com/jfrog/gofrog v1.3.2
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/stretchr/testify v1.8.4
 	github.com/xanzy/ssh-agent v0.3.3

go.sum

@@ -52,10 +52,10 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
 github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
-github.com/jfrog/build-info-go v1.9.15 h1:DN7DKZq6H5FlHfL3Lu8fo4t2INgczRgT09dJiZjJ1oo=
-github.com/jfrog/build-info-go v1.9.15/go.mod h1:XVFk2rCYhIdc7+hIGE8TC3le5PPM+xYHU22udoE2b7Q=
-github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk=
-github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
+github.com/jfrog/build-info-go v1.9.16 h1:zMNxUXve4CZndhlbaEGwgayWPY8CRyPzSobvTKYRPcg=
+github.com/jfrog/build-info-go v1.9.16/go.mod h1:/5VZXH2Ud0IK3cOFwPykjwPOaEcHhzzbjnRiou+YKpM=
+github.com/jfrog/gofrog v1.3.2 h1:TktKP+PdZdxjkYZxWWIq4DkTGSYtr9Slsy+egZpEhUY=
+github.com/jfrog/gofrog v1.3.2/go.mod h1:AQo5Fq0G9nDEF6icH7MYQK0iohR4HuEAXl8jaxRuT6Q=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=

http/httpclient/client.go

@@ -131,8 +131,8 @@ func (jc *HttpClient) Send(method, url string, content []byte, followRedirect, c
 			if resp == nil {
 				return false, errorutils.CheckErrorf("%sReceived empty response from server", logMsgPrefix)
 			}
-			// If response-code < 500, should not retry
-			if resp.StatusCode < 500 {
+			// If response-code < 500 and it is not 429, should not retry
+			if resp.StatusCode < 500 && resp.StatusCode != http.StatusTooManyRequests {
 				return false, nil
 			}
 			// Perform retry

lifecycle/services/distribute.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"github.com/jfrog/jfrog-client-go/auth"
 	"github.com/jfrog/jfrog-client-go/http/jfroghttpclient"
 	"github.com/jfrog/jfrog-client-go/utils/distribution"
@@ -62,7 +63,7 @@ func (dr *DistributeReleaseBundleService) createDistributeBody() ReleaseBundleDi
 	return ReleaseBundleDistributeBody{
 		ReleaseBundleDistributeV1Body: distribution.CreateDistributeV1Body(dr.DistributeParams, dr.DryRun, dr.AutoCreateRepo),
 		Modifications: Modifications{
-			PathMappings: distribution.CreatePathMappings(dr.Pattern, dr.Target),
+			PathMappings: distribution.CreatePathMappingsFromPatternAndTarget(dr.Pattern, dr.Target),
 		},
 	}
 }
@@ -73,7 +74,7 @@ type ReleaseBundleDistributeBody struct {
 }
 
 type Modifications struct {
-	PathMappings []distribution.PathMapping `json:"mappings"`
+	PathMappings []utils.PathMapping `json:"mappings"`
 }
 
 type PathMapping struct {

tests/distribution_test.go

@@ -14,6 +14,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
@@ -52,7 +53,9 @@ func TestDistributionServices(t *testing.T) {
 	t.Run("createSignDistributeDelete", createSignDistributeDelete)
 	t.Run("createSignSyncDistributeDelete", createSignSyncDistributeDelete)
 	t.Run("createDistributeMapping", createDistributeMapping)
-	t.Run("createDistributeMappingPlaceholder", createDistributeMappingPlaceholder)
+	t.Run("createDistributeMappingFromPatternAndTarget", createDistributeMappingFromPatternAndTarget)
+	t.Run("createDistributeMappingWithPlaceholder", createDistributeMappingWithPlaceholder)
+	t.Run("createDistributeMappingFromPatternAndTargetWithPlaceholder", createDistributeMappingFromPatternAndTargetWithPlaceholder)
 
 	artifactoryCleanup(t)
 	deleteGpgKeys(t)
@@ -123,6 +126,9 @@ func createUpdate(t *testing.T) {
 	// Verify was not created.
 	getLocalBundle(t, bundleName, false)
 
+	// Redefine specFiles to create params from scratch
+	createBundleParams.SpecFiles[0] = &utils.CommonParams{Pattern: getRtTargetRepo() + "b.in"}
+
 	// Create unsigned release bundle
 	summary, err := testsBundleCreateService.CreateReleaseBundle(createBundleParams)
 	if !assert.NoError(t, err) {
@@ -147,6 +153,9 @@ func createUpdate(t *testing.T) {
 	// Verify the release bundle was not updated.
 	assertCreatedLocalBundle(t, bundleName, createBundleParams)
 
+	// Redefine specFiles to create params from scratch
+	updateBundleParams.SpecFiles[0] = &utils.CommonParams{Pattern: getRtTargetRepo() + "test/a.in"}
+
 	summary, err = testsBundleUpdateService.UpdateReleaseBundle(updateBundleParams)
 	if !assert.NoError(t, err) {
 		return
@@ -357,12 +366,21 @@ func createDistributeMapping(t *testing.T) {
 
 	// Create release bundle with path mapping from <RtTargetRepo>/b.in to <RtTargetRepo>/b.out
 	createBundleParams := services.NewCreateReleaseBundleParams(bundleName, bundleVersion)
-	createBundleParams.SpecFiles = []*utils.CommonParams{{Pattern: getRtTargetRepo() + "b.in", Target: getRtTargetRepo() + "b.out"}}
+	createBundleParams.SpecFiles = []*utils.CommonParams{
+		{
+			Aql: utils.Aql{
+				ItemsFind: "{\"$or\":[{\"$and\":[{\"repo\":{\"$match\":\"" + strings.TrimSuffix(getRtTargetRepo(), "/") + "\"},\"name\":{\"$match\":\"b.in\"}}]}]}",
+			},
+			PathMapping: utils.PathMapping{
+				Input:  getRtTargetRepo() + "b.in",
+				Output: getRtTargetRepo() + "b.out",
+			},
+		},
+	}
 	createBundleParams.SignImmediately = true
 	summary, err := testsBundleCreateService.CreateReleaseBundle(createBundleParams)
-	if !assert.NoError(t, err) {
-		return
-	}
+	assert.NoError(t, err)
+
 	defer deleteRemoteAndLocalBundle(t, bundleName)
 	assert.NotNil(t, summary)
 	verifyValidSha256(t, summary.GetSha256())
@@ -377,51 +395,85 @@ func createDistributeMapping(t *testing.T) {
 	err = testsBundleDistributeService.Distribute()
 	assert.NoError(t, err)
 
+	// Distribute release bundle
+	assertReleaseBundleDistribution(t, bundleName)
+
 	// Make sure <RtTargetRepo>/b.out does exist in Artifactory
-	searchParams := artifactoryServices.NewSearchParams()
-	searchParams.Pattern = getRtTargetRepo() + "b.out"
-	reader, err := testsSearchService.Search(searchParams)
+	assertFileExistsInArtifactory(t, getRtTargetRepo()+"b.out")
+}
+
+func createDistributeMappingFromPatternAndTarget(t *testing.T) {
+	bundleName := initRemoteDistributionTest(t, "client-test-bundle-"+getRunId())
+
+	// Create release bundle with path mapping from <RtTargetRepo>/b.in to <RtTargetRepo>/b.out
+	createBundleParams := services.NewCreateReleaseBundleParams(bundleName, bundleVersion)
+	createBundleParams.SpecFiles = []*utils.CommonParams{{Pattern: getRtTargetRepo() + "b.in", Target: getRtTargetRepo() + "b.out"}}
+	createBundleParams.SignImmediately = true
+	summary, err := testsBundleCreateService.CreateReleaseBundle(createBundleParams)
 	assert.NoError(t, err)
-	readerCloseAndAssert(t, reader)
-	length, err := reader.Length()
+
+	defer deleteRemoteAndLocalBundle(t, bundleName)
+	assert.NotNil(t, summary)
+	verifyValidSha256(t, summary.GetSha256())
+
+	// Distribute release bundle
+	assertReleaseBundleDistribution(t, bundleName)
+
+	// Make sure <RtTargetRepo>/b.out does exist in Artifactory
+	assertFileExistsInArtifactory(t, getRtTargetRepo()+"b.out")
+}
+
+func createDistributeMappingWithPlaceholder(t *testing.T) {
+	bundleName := initRemoteDistributionTest(t, "client-test-bundle-"+getRunId())
+
+	// Create release bundle with path mapping from <RtTargetRepo>/b.in to <RtTargetRepo>/b.out
+	createBundleParams := services.NewCreateReleaseBundleParams(bundleName, bundleVersion)
+	createBundleParams.SpecFiles = []*utils.CommonParams{
+		{
+			Aql: utils.Aql{
+				ItemsFind: "{\"$or\":[{\"$and\":[{\"repo\":{\"$match\":\"" + strings.TrimSuffix(getRtTargetRepo(), "/") + "\"},\"name\":{\"$match\":\"*.in\"}}]}]}",
+			},
+			PathMapping: utils.PathMapping{
+				Input:  "(" + getRtTargetRepo() + ")" + "(.*).in",
+				Output: "$1$2.out",
+			},
+		},
+	}
+
+	createBundleParams.SignImmediately = true
+	summary, err := testsBundleCreateService.CreateReleaseBundle(createBundleParams)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, length)
+
+	defer deleteRemoteAndLocalBundle(t, bundleName)
+	assert.NotNil(t, summary)
+	verifyValidSha256(t, summary.GetSha256())
+
+	// Distribute release bundle
+	assertReleaseBundleDistribution(t, bundleName)
+
+	// Make sure <RtTargetRepo>/b.out does exist in Artifactory
+	assertFileExistsInArtifactory(t, getRtTargetRepo()+"b.out")
 }
 
-func createDistributeMappingPlaceholder(t *testing.T) {
+func createDistributeMappingFromPatternAndTargetWithPlaceholder(t *testing.T) {
 	bundleName := initRemoteDistributionTest(t, "client-test-bundle-"+getRunId())
 
 	// Create release bundle with path mapping from <RtTargetRepo>/b.in to <RtTargetRepo>/b.out
 	createBundleParams := services.NewCreateReleaseBundleParams(bundleName, bundleVersion)
 	createBundleParams.SpecFiles = []*utils.CommonParams{{Pattern: "(" + getRtTargetRepo() + ")" + "(*).in", Target: "{1}{2}.out"}}
 	createBundleParams.SignImmediately = true
 	summary, err := testsBundleCreateService.CreateReleaseBundle(createBundleParams)
-	if !assert.NoError(t, err) {
-		return
-	}
+	assert.NoError(t, err)
+
 	defer deleteRemoteAndLocalBundle(t, bundleName)
 	assert.NotNil(t, summary)
 	verifyValidSha256(t, summary.GetSha256())
 
 	// Distribute release bundle
-	distributeBundleParams := distribution.NewDistributeReleaseBundleParams(bundleName, bundleVersion)
-	distributeBundleParams.DistributionRules = []*distribution.DistributionCommonParams{{SiteName: "*"}}
-	testsBundleDistributeService.Sync = true
-	// On distribution with path mapping, the target repository cannot be auto-created
-	testsBundleDistributeService.AutoCreateRepo = false
-	testsBundleDistributeService.DistributeParams = distributeBundleParams
-	err = testsBundleDistributeService.Distribute()
-	assert.NoError(t, err)
+	assertReleaseBundleDistribution(t, bundleName)
 
 	// Make sure <RtTargetRepo>/b.out does exist in Artifactory
-	searchParams := artifactoryServices.NewSearchParams()
-	searchParams.Pattern = getRtTargetRepo() + "b.out"
-	reader, err := testsSearchService.Search(searchParams)
-	assert.NoError(t, err)
-	readerCloseAndAssert(t, reader)
-	length, err := reader.Length()
-	assert.NoError(t, err)
-	assert.Equal(t, 1, length)
+	assertFileExistsInArtifactory(t, getRtTargetRepo()+"b.out")
 }
 
 // Send GPG keys to Distribution and Artifactory to allow signing of release bundles
@@ -567,3 +619,26 @@ func deleteRemoteAndLocalBundle(t *testing.T, bundleName string) {
 	artifactoryCleanup(t)
 	assert.NoError(t, err)
 }
+
+func assertFileExistsInArtifactory(t *testing.T, filePath string) {
+	searchParams := artifactoryServices.NewSearchParams()
+	searchParams.Pattern = filePath
+	reader, err := testsSearchService.Search(searchParams)
+	assert.NoError(t, err)
+	readerCloseAndAssert(t, reader)
+	length, err := reader.Length()
+	assert.NoError(t, err)
+	assert.Equal(t, 1, length)
+}
+
+func assertReleaseBundleDistribution(t *testing.T, bundleName string) {
+	// Distribute release bundle
+	distributeBundleParams := distribution.NewDistributeReleaseBundleParams(bundleName, bundleVersion)
+	distributeBundleParams.DistributionRules = []*distribution.DistributionCommonParams{{SiteName: "*"}}
+	testsBundleDistributeService.Sync = true
+	// On distribution with path mapping, the target repository cannot be auto-created
+	testsBundleDistributeService.AutoCreateRepo = false
+	testsBundleDistributeService.DistributeParams = distributeBundleParams
+	err := testsBundleDistributeService.Distribute()
+	assert.NoError(t, err)
+}

utils/distribution/utils.go

@@ -2,19 +2,20 @@ package distribution
 
 import (
 	"github.com/jfrog/gofrog/stringutils"
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"regexp"
 )
 
 var fileSpecCaptureGroup = regexp.MustCompile(`({\d})`)
 
-// Create the path mapping from the input spec
-func CreatePathMappings(pattern, target string) []PathMapping {
+// Create the path mapping from the input spec which includes pattern and target
+func CreatePathMappingsFromPatternAndTarget(pattern, target string) []utils.PathMapping {
 	if len(target) == 0 {
-		return []PathMapping{}
+		return []utils.PathMapping{}
 	}
 
 	// Convert the file spec pattern and target to match the path mapping input and output specifications, respectfully.
-	return []PathMapping{{
+	return []utils.PathMapping{{
 		// The file spec pattern is wildcard based. Convert it to Regex:
 		Input: stringutils.WildcardPatternToRegExp(pattern),
 		// The file spec target contain placeholders-style matching groups, like {1}.
@@ -26,7 +27,14 @@ func CreatePathMappings(pattern, target string) []PathMapping {
 	}}
 }
 
-type PathMapping struct {
-	Input  string `json:"input"`
-	Output string `json:"output"`
+// Create the path mapping from the input spec
+func CreatePathMappings(input, output string) []utils.PathMapping {
+	if len(input) == 0 || len(output) == 0 {
+		return []utils.PathMapping{}
+	}
+
+	return []utils.PathMapping{{
+		Input:  input,
+		Output: output,
+	}}
 }