From 45553c535c74ab1fe22b3769f9cf36040f1d2591 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Thu, 1 Feb 2024 15:45:58 +0200 Subject: [PATCH] Artifactory Release Lifecycle Management - Modify release bundle promotion properties (#2425) --- go.mod | 4 +- go.sum | 8 ++-- lifecycle/cli.go | 11 ++++- lifecycle_test.go | 41 ++++++++++++------- testdata/filespecs/search_all_prod_repo.json | 7 ---- testdata/filespecs/search_all_prod_repo1.json | 7 ++++ testdata/filespecs/search_all_prod_repo2.json | 7 ++++ ...json => prod_repo1_repository_config.json} | 2 +- testdata/prod_repo2_repository_config.json | 6 +++ utils/cliutils/commandsflags.go | 18 +++++--- utils/tests/consts.go | 29 +++++++------ utils/tests/utils.go | 13 +++--- 12 files changed, 100 insertions(+), 53 deletions(-) delete mode 100644 testdata/filespecs/search_all_prod_repo.json create mode 100644 testdata/filespecs/search_all_prod_repo1.json create mode 100644 testdata/filespecs/search_all_prod_repo2.json rename testdata/{prod_repo_repository_config.json => prod_repo1_repository_config.json} (75%) create mode 100644 testdata/prod_repo2_repository_config.json diff --git a/go.mod b/go.mod index 2a25c9d11..798c2de6a 100644 --- a/go.mod +++ b/go.mod @@ -131,9 +131,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240122141213-a1db3c179c7e +replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240201134243-db7eee23bed4 -// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240122123649-74f725715bbe +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240201133918-82ad9eb51c3f // replace github.com/jfrog/jfrog-cli-security => github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac diff --git a/go.sum b/go.sum index 2d978b28b..e0b75738e 100644 --- a/go.sum +++ b/go.sum @@ -140,12 +140,12 @@ github.com/jfrog/gofrog v1.5.1 h1:2AXL8hHu1jJFMIoCqTp2OyRUfEqEp4nC7J8fwn6KtwE= github.com/jfrog/gofrog v1.5.1/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-cli-core/v2 v2.47.11 h1:2oHyITZjnjuuwU/LEVedlA3NkJz3PbaXdBT2Y1kt21o= -github.com/jfrog/jfrog-cli-core/v2 v2.47.11/go.mod h1:RVn4pIkR5fPUnr8gFXt61ou3pCNrrDdRQUpcolP4lhw= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240201134243-db7eee23bed4 h1:fXSEae7dKSJo1n4BCVypvSHYNB9gDyfRwmv6RTahuk8= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240201134243-db7eee23bed4/go.mod h1:reOjNxDdVFnBmv3xu8+19eAegxkVRCpWAl6RZvtL6p8= github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac h1:tNn3TQXaIJZ9Fu5jiVB9lWpJAKkEGWNjz/6WzHhHePI= github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac/go.mod h1:X4rz1639L8vWKJgpLxpO3ddkIW7KaCaQjbwani7FPf4= -github.com/jfrog/jfrog-client-go v1.36.0 h1:iODLDjYSlK7rLH8/lEmAFHwYsboeBfaqxXybz6waraE= -github.com/jfrog/jfrog-client-go v1.36.0/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240201133918-82ad9eb51c3f h1:mbWuP+LpyOKJI7FLFdauGG0SEQQ6KcoA4CwOAgsiYg8= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240201133918-82ad9eb51c3f/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU= github.com/jszwec/csvutil v1.9.0 h1:iTmq9G1P0e+AUq/MkFg6tetJ+1BH3fOX8Xi0RAcwiGc= github.com/jszwec/csvutil v1.9.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/lifecycle/cli.go b/lifecycle/cli.go index e4b29edbd..a27268cb8 100644 --- a/lifecycle/cli.go +++ b/lifecycle/cli.go @@ -16,6 +16,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/urfave/cli" + "strings" ) const lcCategory = "Lifecycle" @@ -119,7 +120,8 @@ func promote(c *cli.Context) error { createCmd := lifecycle.NewReleaseBundlePromoteCommand().SetServerDetails(lcDetails).SetReleaseBundleName(c.Args().Get(0)). SetReleaseBundleVersion(c.Args().Get(1)).SetEnvironment(c.Args().Get(2)).SetSigningKeyName(c.String(cliutils.SigningKey)). - SetSync(c.Bool(cliutils.Sync)).SetReleaseBundleProject(cliutils.GetProject(c)).SetOverwrite(c.Bool(cliutils.Overwrite)) + SetSync(c.Bool(cliutils.Sync)).SetReleaseBundleProject(cliutils.GetProject(c)). + SetIncludeReposPatterns(splitRepos(c, cliutils.IncludeRepos)).SetExcludeReposPatterns(splitRepos(c, cliutils.ExcludeRepos)) return commands.Exec(createCmd) } @@ -186,3 +188,10 @@ func PlatformToLifecycleUrls(lcDetails *coreConfig.ServerDetails) { lcDetails.LifecycleUrl = utils.AddTrailingSlashIfNeeded(lcDetails.Url) + "lifecycle/" lcDetails.Url = "" } + +func splitRepos(c *cli.Context, reposOptionKey string) []string { + if c.IsSet(reposOptionKey) { + return strings.Split(c.String(reposOptionKey), ";") + } + return nil +} diff --git a/lifecycle_test.go b/lifecycle_test.go index 1519f0197..a62ff6f34 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -15,7 +15,6 @@ import ( "github.com/jfrog/jfrog-client-go/lifecycle" "github.com/jfrog/jfrog-client-go/lifecycle/services" clientUtils "github.com/jfrog/jfrog-client-go/utils" - "github.com/jfrog/jfrog-client-go/utils/distribution" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/stretchr/testify/assert" "net/http" @@ -63,21 +62,29 @@ func TestLifecycle(t *testing.T) { createRb(t, releaseBundlesSpec, cliutils.ReleaseBundles, tests.LcRbName3, number3, true) defer deleteReleaseBundle(t, lcManager, tests.LcRbName3, number3) - // Promote the last release bundle. + // Promote the last release bundle to prod repo 1. promoteRb(t, lcManager, number3) - // Verify the artifacts of both the initial release bundles made it to the prod repo. - searchProdSpec, err := tests.CreateSpec(tests.SearchAllProdRepo) - assert.NoError(t, err) - inttestutils.VerifyExistInArtifactory(tests.GetExpectedLifecycleArtifacts(), searchProdSpec, serverDetails, t) + // Assert the artifacts of both the initial release bundles made it to prod repo 1. + assertExpectedArtifacts(t, tests.SearchAllProdRepo1, tests.GetExpectedLifecycleArtifacts()) + // Assert no artifacts were promoted to prod repo 2. + assertExpectedArtifacts(t, tests.SearchAllProdRepo2, []string{}) + + // TODO Temporarily disabling till distribution on testing suite is stable. + /* + distributeRb(t) + defer remoteDeleteReleaseBundle(t, lcManager, tests.LcRbName3, number3) - distributeRb(t) - defer remoteDeleteReleaseBundle(t, lcManager, tests.LcRbName3, number3) + // Verify the artifacts were distributed correctly by the provided path mappings. + assertExpectedArtifacts(t, tests.SearchAllDevRepo, tests.GetExpectedLifecycleDistributedArtifacts()) + */ - // Verify the artifacts were distributed correctly by the provided path mappings. - searchDevSpec, err := tests.CreateSpec(tests.SearchAllDevRepo) +} + +func assertExpectedArtifacts(t *testing.T, specFileName string, expected []string) { + searchProdSpec, err := tests.CreateSpec(specFileName) assert.NoError(t, err) - inttestutils.VerifyExistInArtifactory(tests.GetExpectedLifecycleDistributedArtifacts(), searchDevSpec, serverDetails, t) + inttestutils.VerifyExistInArtifactory(expected, searchProdSpec, serverDetails, t) } func uploadBuilds(t *testing.T) func() { @@ -108,6 +115,7 @@ func createRb(t *testing.T, specName, sourceOption, rbName, rbVersion string, sy assert.NoError(t, lcCli.Exec(argsAndOptions...)) } +/* func distributeRb(t *testing.T) { distributionRulesPath := filepath.Join(tests.GetTestResourcesPath(), "distribution", tests.DistributionRules) assert.NoError(t, lcCli.Exec( @@ -119,6 +127,7 @@ func distributeRb(t *testing.T) { // Wait after distribution before asserting. Can be removed once distribute supports sync. time.Sleep(5 * time.Second) } +*/ func getOption(option, value string) string { return fmt.Sprintf("--%s=%s", option, value) @@ -127,7 +136,7 @@ func getOption(option, value string) string { func promoteRb(t *testing.T, lcManager *lifecycle.LifecycleServicesManager, rbVersion string) { output := lcCli.RunCliCmdWithOutput(t, "rbp", tests.LcRbName3, rbVersion, prodEnvironment, getOption(cliutils.SigningKey, gpgKeyPairName), - getOption(cliutils.Overwrite, "true"), + getOption(cliutils.IncludeRepos, tests.RtProdRepo1), "--project=default") var promotionResp services.RbPromotionResp if !assert.NoError(t, json.Unmarshal([]byte(output), &promotionResp)) { @@ -192,11 +201,12 @@ func deleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServicesMan ReleaseBundleVersion: rbVersion, } - assert.NoError(t, lcManager.DeleteReleaseBundle(rbDetails, services.ReleaseBundleQueryParams{Async: false})) + assert.NoError(t, lcManager.DeleteReleaseBundle(rbDetails, services.CommonOptionalQueryParams{Async: false})) // Wait after remote deleting. Can be removed once remote deleting supports sync. time.Sleep(5 * time.Second) } +/* func remoteDeleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServicesManager, rbName, rbVersion string) { params := distribution.NewDistributeReleaseBundleParams(rbName, rbVersion) rules := &distribution.DistributionCommonParams{ @@ -211,6 +221,8 @@ func remoteDeleteReleaseBundle(t *testing.T, lcManager *lifecycle.LifecycleServi time.Sleep(5 * time.Second) } +*/ + func uploadBuild(t *testing.T, specFileName, buildName, buildNumber string) { specFile, err := tests.CreateSpec(specFileName) assert.NoError(t, err) @@ -264,7 +276,8 @@ func CleanLifecycleTests() { func cleanLifecycleTests(t *testing.T) { deleteFilesFromRepo(t, tests.RtDevRepo) - deleteFilesFromRepo(t, tests.RtProdRepo) + deleteFilesFromRepo(t, tests.RtProdRepo1) + deleteFilesFromRepo(t, tests.RtProdRepo2) tests.CleanFileSystem() } diff --git a/testdata/filespecs/search_all_prod_repo.json b/testdata/filespecs/search_all_prod_repo.json deleted file mode 100644 index 2930acf50..000000000 --- a/testdata/filespecs/search_all_prod_repo.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files": [ - { - "pattern": "${PROD_REPO}/*" - } - ] -} \ No newline at end of file diff --git a/testdata/filespecs/search_all_prod_repo1.json b/testdata/filespecs/search_all_prod_repo1.json new file mode 100644 index 000000000..78bf1ac12 --- /dev/null +++ b/testdata/filespecs/search_all_prod_repo1.json @@ -0,0 +1,7 @@ +{ + "files": [ + { + "pattern": "${PROD_REPO1}/*" + } + ] +} \ No newline at end of file diff --git a/testdata/filespecs/search_all_prod_repo2.json b/testdata/filespecs/search_all_prod_repo2.json new file mode 100644 index 000000000..07f4ed213 --- /dev/null +++ b/testdata/filespecs/search_all_prod_repo2.json @@ -0,0 +1,7 @@ +{ + "files": [ + { + "pattern": "${PROD_REPO2}/*" + } + ] +} \ No newline at end of file diff --git a/testdata/prod_repo_repository_config.json b/testdata/prod_repo1_repository_config.json similarity index 75% rename from testdata/prod_repo_repository_config.json rename to testdata/prod_repo1_repository_config.json index 6187c443e..8a21e0d1a 100644 --- a/testdata/prod_repo_repository_config.json +++ b/testdata/prod_repo1_repository_config.json @@ -1,5 +1,5 @@ { - "key": "${PROD_REPO}", + "key": "${PROD_REPO1}", "rclass": "local", "packageType": "generic", "environments":["PROD"] diff --git a/testdata/prod_repo2_repository_config.json b/testdata/prod_repo2_repository_config.json new file mode 100644 index 000000000..76aba6ecd --- /dev/null +++ b/testdata/prod_repo2_repository_config.json @@ -0,0 +1,6 @@ +{ + "key": "${PROD_REPO2}", + "rclass": "local", + "packageType": "generic", + "environments":["PROD"] +} diff --git a/utils/cliutils/commandsflags.go b/utils/cliutils/commandsflags.go index d110978ec..9946e24cf 100644 --- a/utils/cliutils/commandsflags.go +++ b/utils/cliutils/commandsflags.go @@ -585,12 +585,13 @@ const ( lcReleaseBundles = lifecyclePrefix + ReleaseBundles SigningKey = "signing-key" lcSigningKey = lifecyclePrefix + SigningKey - lcOverwrite = lifecyclePrefix + Overwrite PathMappingPattern = "mapping-pattern" lcPathMappingPattern = lifecyclePrefix + PathMappingPattern PathMappingTarget = "mapping-target" lcPathMappingTarget = lifecyclePrefix + PathMappingTarget lcDryRun = lifecyclePrefix + dryRun + lcIncludeRepos = lifecyclePrefix + IncludeRepos + lcExcludeRepos = lifecyclePrefix + ExcludeRepos ) var flagsMap = map[string]cli.Flag{ @@ -1651,10 +1652,6 @@ var flagsMap = map[string]cli.Flag{ Name: SigningKey, Usage: "[Mandatory] The GPG/RSA key-pair name given in Artifactory.` `", }, - lcOverwrite: cli.BoolFlag{ - Name: Overwrite, - Usage: "[Default: false] Set to true to replace artifacts with the same name but a different checksum if such already exist at the promotion targets. By default, the promotion is stopped in a case of such conflict.` `", - }, lcPathMappingPattern: cli.StringFlag{ Name: PathMappingPattern, Usage: "[Optional] Specify along with '" + PathMappingTarget + "' to distribute artifacts to a different path on the edge node. You can use wildcards to specify multiple artifacts.` `", @@ -1673,6 +1670,15 @@ var flagsMap = map[string]cli.Flag{ Usage: "Default: false] [npm] when set, the Contextual Analysis scan also uses the code of the project dependencies to determine the applicability of the vulnerability.", Hidden: true, }, + lcIncludeRepos: cli.StringFlag{ + Name: IncludeRepos, + Usage: "[Optional] A list of semicolon-separated repositories to include in the promotion. If this property is left undefined, all repositories (except those specifically excluded) are included in the promotion. " + + "If one or more repositories are specifically included, all other repositories are excluded.` `", + }, + lcExcludeRepos: cli.StringFlag{ + Name: ExcludeRepos, + Usage: "[Optional] A list of semicolon-separated repositories to exclude from the promotion.` `", + }, atcProject: cli.StringFlag{ Name: Project, Usage: "[Optional] The project for which this token is created. Enter the project name on which you want to apply this token.` `", @@ -1999,7 +2005,7 @@ var commandFlags = map[string][]string{ lcUrl, user, password, accessToken, serverId, lcSigningKey, lcSync, lcProject, lcBuilds, lcReleaseBundles, }, ReleaseBundlePromote: { - lcUrl, user, password, accessToken, serverId, lcSigningKey, lcSync, lcProject, lcOverwrite, + lcUrl, user, password, accessToken, serverId, lcSigningKey, lcSync, lcProject, lcIncludeRepos, lcExcludeRepos, }, ReleaseBundleDistribute: { lcUrl, user, password, accessToken, serverId, lcDryRun, DistRules, site, city, countryCodes, diff --git a/utils/tests/consts.go b/utils/tests/consts.go index 160ef87a3..8067a71dd 100644 --- a/utils/tests/consts.go +++ b/utils/tests/consts.go @@ -92,7 +92,8 @@ const ( SearchAllNpm = "search_all_npm.json" SearchAllRepo1 = "search_all_repo1.json" SearchAllDevRepo = "search_all_dev_repo.json" - SearchAllProdRepo = "search_all_prod_repo.json" + SearchAllProdRepo1 = "search_all_prod_repo1.json" + SearchAllProdRepo2 = "search_all_prod_repo2.json" SearchDistRepoByInSuffix = "search_dist_repo_by_in_suffix.json" SearchRepo1ByInSuffix = "search_repo1_by_in_suffix.json" GoPublishRepoExcludes = "go_publish_repo_excludes.json" @@ -150,7 +151,8 @@ const ( DockerVirtualRepositoryConfig = "docker_virtual_repository_config.json" XrayEndpoint = "xray/" DevRepoRepositoryConfig = "dev_repo_repository_config.json" - ProdRepoRepositoryConfig = "prod_repo_repository_config.json" + ProdRepo1RepositoryConfig = "prod_repo1_repository_config.json" + ProdRepo2RepositoryConfig = "prod_repo2_repository_config.json" UploadDevSpecA = "upload_dev_spec_a.json" UploadDevSpecB = "upload_dev_spec_b.json" UploadDevSpecC = "upload_dev_spec_c.json" @@ -187,8 +189,9 @@ var ( RtRepo2 = "cli-rt2" RtVirtualRepo = "cli-rt-virtual" // Repositories that are assigned to an environment. - RtDevRepo = "cli-rt-dev" - RtProdRepo = "cli-rt-prod" + RtDevRepo = "cli-rt-dev" + RtProdRepo1 = "cli-rt-prod1" + RtProdRepo2 = "cli-rt-prod2" // These are not actual repositories. These patterns are meant to be used in both Repo1 and Repo2. RtRepo1And2 = "cli-rt*" RtRepo1And2Placeholder = "cli-rt(*)" @@ -2088,15 +2091,15 @@ func GetTransferExpectedRepoSnapshot() []string { func GetExpectedLifecycleArtifacts() []string { return []string{ - RtProdRepo + "/a1.in", - RtProdRepo + "/a2.in", - RtProdRepo + "/a3.in", - RtProdRepo + "/b1.in", - RtProdRepo + "/b2.in", - RtProdRepo + "/b3.in", - RtProdRepo + "/c1.in", - RtProdRepo + "/c2.in", - RtProdRepo + "/c3.in", + RtProdRepo1 + "/a1.in", + RtProdRepo1 + "/a2.in", + RtProdRepo1 + "/a3.in", + RtProdRepo1 + "/b1.in", + RtProdRepo1 + "/b2.in", + RtProdRepo1 + "/b3.in", + RtProdRepo1 + "/c1.in", + RtProdRepo1 + "/c2.in", + RtProdRepo1 + "/c3.in", } } diff --git a/utils/tests/utils.go b/utils/tests/utils.go index d113ddcff..5037e5bac 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + "github.com/jfrog/jfrog-client-go/utils/tests" "io" "math/rand" "os" @@ -36,7 +37,6 @@ import ( clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" ) @@ -274,7 +274,8 @@ var reposConfigMap = map[*string]string{ &DockerRemoteRepo: DockerRemoteRepositoryConfig, &DockerVirtualRepo: DockerVirtualRepositoryConfig, &RtDevRepo: DevRepoRepositoryConfig, - &RtProdRepo: ProdRepoRepositoryConfig, + &RtProdRepo1: ProdRepo1RepositoryConfig, + &RtProdRepo2: ProdRepo2RepositoryConfig, } var CreatedNonVirtualRepositories map[*string]string @@ -324,7 +325,7 @@ func GetNonVirtualRepositories() map[*string]string { TestXray: {&NpmRemoteRepo, &NugetRemoteRepo, &YarnRemoteRepo, &GradleRemoteRepo, &MvnRemoteRepo, &GoRepo, &GoRemoteRepo, &PypiRemoteRepo}, TestAccess: {&RtRepo1}, TestTransfer: {&RtRepo1, &RtRepo2, &MvnRepo1, &MvnRemoteRepo, &DockerRemoteRepo}, - TestLifecycle: {&RtDevRepo, &RtProdRepo}, + TestLifecycle: {&RtDevRepo, &RtProdRepo1, &RtProdRepo2}, } return getNeededRepositories(nonVirtualReposMap) } @@ -442,7 +443,8 @@ func getSubstitutionMap() map[string]string { "${RB_NAME1}": LcRbName1, "${RB_NAME2}": LcRbName2, "${DEV_REPO}": RtDevRepo, - "${PROD_REPO}": RtProdRepo, + "${PROD_REPO1}": RtProdRepo1, + "${PROD_REPO2}": RtProdRepo2, } } @@ -492,7 +494,8 @@ func AddTimestampToGlobalVars() { RtRepo2 += uniqueSuffix RtVirtualRepo += uniqueSuffix RtDevRepo += uniqueSuffix - RtProdRepo += uniqueSuffix + RtProdRepo1 += uniqueSuffix + RtProdRepo2 += uniqueSuffix // Builds/bundles/images BundleName += uniqueSuffix