Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve get filtered repositories code #1280

Merged
merged 48 commits into from
Oct 10, 2024
Merged
Show file tree
Hide file tree
Changes from 47 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
bd80ddd
metadata
sverdlov93 Aug 25, 2024
596a766
metadata
sverdlov93 Aug 25, 2024
83cdb96
Merge branch 'dev' into fix-template
sverdlov93 Aug 26, 2024
130c814
Merge branch 'dev' into fix-template
sverdlov93 Aug 26, 2024
3565983
metadata
sverdlov93 Aug 28, 2024
511132a
Merge remote-tracking branch 'sverdlov93/fix-template' into fix-template
sverdlov93 Aug 28, 2024
e13f31e
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Aug 28, 2024
e92075b
metadata
sverdlov93 Aug 28, 2024
feac980
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Aug 29, 2024
0646e8e
metadata
sverdlov93 Aug 29, 2024
42686b8
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Sep 5, 2024
12bee92
metadata
sverdlov93 Sep 5, 2024
ea701bf
metadata
sverdlov93 Sep 5, 2024
2a13aa6
Remove redundant log
sverdlov93 Sep 8, 2024
cac58da
Remove redundant log
sverdlov93 Sep 8, 2024
3a3fe87
Remove redundant log
sverdlov93 Sep 8, 2024
ed6b3d2
Remove redundant log
sverdlov93 Sep 8, 2024
45b1e18
Remove redundant log
sverdlov93 Sep 8, 2024
c5eaad2
Remove redundant log
sverdlov93 Sep 8, 2024
0d0241a
Remove redundant log
sverdlov93 Sep 8, 2024
9f04ee3
Remove redundant log
sverdlov93 Sep 9, 2024
9906505
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Sep 9, 2024
2e464c7
Remove redundant log
sverdlov93 Sep 9, 2024
dcd034a
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Sep 9, 2024
099ae81
Remove redundant log
sverdlov93 Sep 9, 2024
5856ad9
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Sep 12, 2024
4edd481
Remove redundant log
sverdlov93 Sep 12, 2024
b612100
Remove redundant log
sverdlov93 Sep 12, 2024
5c3a7f7
Remove redundant log
sverdlov93 Sep 12, 2024
d6dbc56
Remove redundant log
sverdlov93 Sep 12, 2024
d3bfe11
Remove redundant log
sverdlov93 Sep 12, 2024
6e90b66
Remove redundant log
sverdlov93 Sep 15, 2024
f1c2bbd
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Sep 17, 2024
41aa3e8
Remove redundant log
sverdlov93 Sep 17, 2024
64b7b91
Remove redundant log
sverdlov93 Sep 17, 2024
b205894
Remove redundant log
sverdlov93 Sep 17, 2024
3c71af3
Remove redundant log
sverdlov93 Sep 18, 2024
24ef29a
Remove redundant log
sverdlov93 Oct 7, 2024
ba8bebc
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Oct 7, 2024
3691e30
Remove redundant log
sverdlov93 Oct 7, 2024
b72be6c
Remove redundant log
sverdlov93 Oct 7, 2024
e99595c
Remove redundant log
sverdlov93 Oct 7, 2024
8444dbb
Remove redundant log
sverdlov93 Oct 7, 2024
d2207be
Merge branch 'dev' of https://github.com/jfrog/jfrog-cli-core into fi…
sverdlov93 Oct 8, 2024
fc7a8e6
Improve repositores code
sverdlov93 Oct 10, 2024
db9e0ca
Improve repositores code
sverdlov93 Oct 10, 2024
d9064a0
Improve repositores code
sverdlov93 Oct 10, 2024
6dbf522
Update common/commands/configfile.go
sverdlov93 Oct 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions artifactory/commands/transferfiles/transfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"github.com/jfrog/jfrog-client-go/artifactory/services"
"os"
"os/signal"
"path/filepath"
Expand Down Expand Up @@ -342,11 +343,11 @@ func (tdc *TransferFilesCommand) NewTransferDataPreChecksRunner() (runner *prech
if err != nil {
return
}
localRepos, err := utils.GetFilteredRepositoriesByNameAndType(serviceManager, tdc.includeReposPatterns, tdc.excludeReposPatterns, utils.Local)
localRepos, err := utils.GetFilteredRepositoriesWithFilterParams(serviceManager, tdc.includeReposPatterns, tdc.excludeReposPatterns, services.RepositoriesFilterParams{RepoType: utils.Local.String()})
if err != nil {
return
}
federatedRepos, err := utils.GetFilteredRepositoriesByNameAndType(serviceManager, tdc.includeReposPatterns, tdc.excludeReposPatterns, utils.Federated)
federatedRepos, err := utils.GetFilteredRepositoriesWithFilterParams(serviceManager, tdc.includeReposPatterns, tdc.excludeReposPatterns, services.RepositoriesFilterParams{RepoType: utils.Federated.String()})
if err != nil {
return
}
Expand Down Expand Up @@ -586,11 +587,11 @@ func (tdc *TransferFilesCommand) getAllLocalRepos(serverDetails *config.ServerDe
}
excludeRepoPatternsWithBuildInfo := tdc.excludeReposPatterns
excludeRepoPatternsWithBuildInfo = append(excludeRepoPatternsWithBuildInfo, "*-build-info")
localRepos, err := utils.GetFilteredRepositoriesByNameAndType(serviceManager, tdc.includeReposPatterns, excludeRepoPatternsWithBuildInfo, utils.Local)
localRepos, err := utils.GetFilteredRepositoriesWithFilterParams(serviceManager, tdc.includeReposPatterns, excludeRepoPatternsWithBuildInfo, services.RepositoriesFilterParams{RepoType: utils.Local.String()})
if err != nil {
return []string{}, []string{}, err
}
federatedRepos, err := utils.GetFilteredRepositoriesByNameAndType(serviceManager, tdc.includeReposPatterns, excludeRepoPatternsWithBuildInfo, utils.Federated)
federatedRepos, err := utils.GetFilteredRepositoriesWithFilterParams(serviceManager, tdc.includeReposPatterns, excludeRepoPatternsWithBuildInfo, services.RepositoriesFilterParams{RepoType: utils.Federated.String()})
if err != nil {
return []string{}, []string{}, err
}
Expand Down
74 changes: 18 additions & 56 deletions artifactory/utils/repositoryutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (
"path"
"strings"

"github.com/jfrog/jfrog-cli-core/v2/utils/config"

"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/artifactory/services"
clientUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
Expand Down Expand Up @@ -57,26 +55,6 @@ var blacklistedRepositories = []string{
"jfrog-usage-logs", "jfrog-billing-logs", "jfrog-logs", "artifactory-pipe-info", "auto-trashcan", "jfrog-support-bundle", "_intransit", "artifactory-edge-uploads",
}

// GetRepositories returns the names of local, remote, virtual or federated repositories filtered by their type.
// artDetails - Artifactory server details
// repoTypes - Repository types to filter. If empty - return all repository types.
func GetRepositories(artDetails *config.ServerDetails, repoTypes ...RepoType) ([]string, error) {
sm, err := CreateServiceManager(artDetails, 3, 0, false)
if err != nil {
return nil, err
}
repos := []string{}
for _, repoType := range repoTypes {
filteredRepos, err := GetFilteredRepositoriesByNameAndType(sm, nil, nil, repoType)
if err != nil {
return repos, err
}
repos = append(repos, filteredRepos...)
}

return repos, nil
}

// Since we can't search dependencies in a remote repository, we will turn the search to the repository's cache.
// Local/Virtual repository name will be returned as is.
func GetRepoNameForDependenciesSearch(repoName string, serviceManager artifactory.ArtifactoryServicesManager) (string, error) {
Expand All @@ -99,44 +77,27 @@ func IsRemoteRepo(repoName string, serviceManager artifactory.ArtifactoryService
return repoDetails.GetRepoType() == "remote", nil
}

// GetFilteredRepositoriesByName returns the names of local, remote, virtual and federated repositories filtered by their names.
// includePatterns - patterns of repository names (can contain wildcards) to include in the results. A repository's name
// must match at least one of these patterns in order to be included in the results. If includePatterns' length is zero,
// all repositories are included.
// excludePatterns - patterns of repository names (can contain wildcards) to exclude from the results. A repository's name
// must NOT match any of these patterns in order to be included in the results.
func GetFilteredRepositoriesByName(servicesManager artifactory.ArtifactoryServicesManager, includePatterns, excludePatterns []string) ([]string, error) {
repoDetailsList, err := servicesManager.GetAllRepositories()
if err != nil {
return nil, err
}

return getFilteredRepositories(repoDetailsList, includePatterns, excludePatterns)
}

// GetFilteredRepositoriesByNameAndType returns the names of local, remote, virtual and federated repositories filtered by their names and type.
// includePatterns - patterns of repository names (can contain wildcards) to include in the results. A repository's name
// GetFilteredRepositoriesWithFilterParams returns the names of local, remote, virtual, and federated repositories filtered by their names and type.
// servicesManager - The Artifactory services manager used to interact with the Artifactory server.
// includePatterns - Patterns of repository names (can contain wildcards) to include in the results. A repository's name
// must match at least one of these patterns in order to be included in the results. If includePatterns' length is zero,
// all repositories are included.
// excludePatterns - patterns of repository names (can contain wildcards) to exclude from the results. A repository's name
// excludePatterns - Patterns of repository names (can contain wildcards) to exclude from the results. A repository's name
// must NOT match any of these patterns in order to be included in the results.
// repoType - only repositories of this type will be returned.
func GetFilteredRepositoriesByNameAndType(servicesManager artifactory.ArtifactoryServicesManager, includePatterns, excludePatterns []string, repoType RepoType) ([]string, error) {
repoDetailsList, err := servicesManager.GetAllRepositoriesFiltered(services.RepositoriesFilterParams{RepoType: repoType.String()})
// filterParams - Parameters to filter the repositories by their type.
// Returns a slice of repository names that match the given patterns and type, or an error if the operation fails.
func GetFilteredRepositoriesWithFilterParams(servicesManager artifactory.ArtifactoryServicesManager, includePatterns, excludePatterns []string, filterParams services.RepositoriesFilterParams) ([]string, error) {
repoDetailsList, err := servicesManager.GetAllRepositoriesFiltered(filterParams)
if err != nil {
return nil, err
}

return getFilteredRepositories(repoDetailsList, includePatterns, excludePatterns)
}

func getFilteredRepositories(repoDetailsList *[]services.RepositoryDetails, includePatterns, excludePatterns []string) ([]string, error) {
var repoKeys []string
for _, repoDetails := range *repoDetailsList {
repoKeys = append(repoKeys, repoDetails.Key)
repoKeys := make([]string, len(*repoDetailsList))
for i, repoDetails := range *repoDetailsList {
repoKeys[i] = repoDetails.Key
}

return filterRepositoryNames(&repoKeys, includePatterns, excludePatterns)
return filterRepositoryNames(repoKeys, includePatterns, excludePatterns)
}

// GetFilteredBuildInfoRepositories returns the names of all build-info repositories filtered by their names.
Expand All @@ -147,26 +108,27 @@ func getFilteredRepositories(repoDetailsList *[]services.RepositoryDetails, incl
// excludePatterns - patterns of repository names (can contain wildcards) to exclude from the results. A repository's name
// must NOT match any of these patterns in order to be included in the results.
func GetFilteredBuildInfoRepositories(storageInfo *clientUtils.StorageInfo, includePatterns, excludePatterns []string) ([]string, error) {
var repoKeys []string
repoKeys := make([]string, 0, len(storageInfo.RepositoriesSummaryList))

for _, repoSummary := range storageInfo.RepositoriesSummaryList {
if strings.ToLower(repoSummary.PackageType) == buildInfoPackageType {
repoKeys = append(repoKeys, repoSummary.RepoKey)
}
}
return filterRepositoryNames(&repoKeys, includePatterns, excludePatterns)
return filterRepositoryNames(repoKeys, includePatterns, excludePatterns)
}

// Filter repositories by name and return a list of repository names
// repos - The repository keys to filter
// includePatterns - Repositories inclusion wildcard pattern
// excludePatterns - Repositories exclusion wildcard pattern
func filterRepositoryNames(repoKeys *[]string, includePatterns, excludePatterns []string) ([]string, error) {
func filterRepositoryNames(repoKeys []string, includePatterns, excludePatterns []string) ([]string, error) {
includedRepos := datastructures.MakeSet[string]()
includeExcludeFilter := &IncludeExcludeFilter{
includeExcludeFilter := IncludeExcludeFilter{
IncludePatterns: includePatterns,
ExcludePatterns: excludePatterns,
}
for _, repoKey := range *repoKeys {
for _, repoKey := range repoKeys {
repoIncluded, err := includeExcludeFilter.ShouldIncludeRepository(repoKey)
if err != nil {
return nil, err
Expand Down
2 changes: 1 addition & 1 deletion artifactory/utils/repositoryutils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestFilterRepositoryNames(t *testing.T) {
"jfrog-npm-local",
"jfrog-generic-remote",
}
actualRepoNames, err := filterRepositoryNames(&repos, includePatterns, excludePatterns)
actualRepoNames, err := filterRepositoryNames(repos, includePatterns, excludePatterns)
assert.NoError(t, err)
assert.ElementsMatch(t, expectedRepoNames, actualRepoNames)
}
Expand Down
20 changes: 17 additions & 3 deletions common/commands/configfile.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package commands

import (
"fmt"
"github.com/jfrog/jfrog-client-go/artifactory/services"
"os"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -134,10 +136,10 @@ func createBuildConfig(global bool, confType project.ProjectType, configFile *Co
}
// Populate, validate and write the config file
configFilePath := filepath.Join(projectDir, confType.String()+".yaml")
if err := configFile.VerifyConfigFile(configFilePath); err != nil {
if err = configFile.VerifyConfigFile(configFilePath); err != nil {
return err
}
if err := handleInteractiveConfigCreation(configFile, confType); err != nil {
if err = handleInteractiveConfigCreation(configFile, confType); err != nil {
return err
}
if err = configFile.validateConfig(); err != nil {
Expand Down Expand Up @@ -649,7 +651,19 @@ func getRepositories(serverId string, repoTypes ...utils.RepoType) ([]string, er
return nil, err
}

return utils.GetRepositories(artDetails, repoTypes...)
sm, err := utils.CreateServiceManager(artDetails, 3, 0, false)
if err != nil {
return nil, err
}
repos := []string{}
for _, repoType := range repoTypes {
filteredRepos, err := utils.GetFilteredRepositoriesWithFilterParams(sm, nil, nil, services.RepositoriesFilterParams{RepoType: repoType.String()})
if err != nil {
return nil, fmt.Errorf("failed getting %s repositories list: %w", repoType, err)
sverdlov93 marked this conversation as resolved.
Show resolved Hide resolved
}
repos = append(repos, filteredRepos...)
}
return repos, nil
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just moved from repoutils to here

}

func defaultIfNotSet(c *cli.Context, flagName string, defaultValue string) string {
Expand Down
Loading