From 7cd8869256b93c7413c8f70efe634488718828d1 Mon Sep 17 00:00:00 2001 From: victorujfrog <110019886+victorujfrog@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:11:28 +0200 Subject: [PATCH 1/3] Add cdnRedirect flag to local and remote repositories base params (#858) --- artifactory/services/repository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/artifactory/services/repository.go b/artifactory/services/repository.go index d15caaaed..eb7e84702 100644 --- a/artifactory/services/repository.go +++ b/artifactory/services/repository.go @@ -77,6 +77,7 @@ type AdditionalRepositoryBaseParams struct { PropertySets []string `json:"propertySets,omitempty"` DownloadRedirect *bool `json:"downloadRedirect,omitempty"` PriorityResolution *bool `json:"priorityResolution,omitempty"` + CdnRedirect *bool `json:"cdnRedirect,omitempty"` } type CargoRepositoryParams struct { From 00d80f604090ad5c20d0d81d1a144c74f3893883 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Thu, 9 Nov 2023 12:58:22 +0200 Subject: [PATCH 2/3] New AQL query for build-info json (#854) --- artifactory/services/utils/aqlquerybuilder.go | 56 ++++++++++++++++++- .../services/utils/aqlquerybuilder_test.go | 33 +++++++++++ tests/artifactoryaql_test.go | 45 +++++++++++++++ tests/jfrogclient_test.go | 1 + tests/utils_test.go | 23 +++++--- tests/xraybinmgr_test.go | 1 + tests/xraywatch_test.go | 2 + 7 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 tests/artifactoryaql_test.go diff --git a/artifactory/services/utils/aqlquerybuilder.go b/artifactory/services/utils/aqlquerybuilder.go index caaec0c9c..304715e64 100644 --- a/artifactory/services/utils/aqlquerybuilder.go +++ b/artifactory/services/utils/aqlquerybuilder.go @@ -2,14 +2,30 @@ package utils import ( "fmt" - "golang.org/x/exp/slices" "strconv" "strings" + "unicode" + + "golang.org/x/exp/slices" "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" ) +const spaceEncoding = "%20" + +var specialAqlCharacters = map[rune]string{ + '/': "%2F", + '\\': "%5C", + '|': "%7C", + '*': "%2A", + '?': "%3F", + '\'': "%22", + ':': "%3A", + ';': "%3B", + '%': "%25", +} + // Returns an AQL body string to search file in Artifactory by pattern, according the specified arguments requirements. func CreateAqlBodyForSpecWithPattern(params *CommonParams) (string, error) { searchPattern := prepareSourceSearchPattern(params.Pattern, params.Target) @@ -149,6 +165,44 @@ func CreateAqlQueryForPypi(repo, file string) string { return fmt.Sprintf(itemsPart, repo, file, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_md5", "actual_sha1", "sha256"})) } +// noinspection GoUnusedExportedFunction +func CreateAqlQueryForBuildInfoJson(project, buildName, buildNumber, timestamp string) string { + if project == "" { + project = "artifactory" + } else { + project = encodeForBuildInfoRepository(project) + } + itemsPart := + `items.find({ + "repo": "%s", + "path": { + "$match": "%s" + }, + "name": { + "$match": "%s-%s.json" + } + })%s` + return fmt.Sprintf(itemsPart, project+"-build-info", encodeForBuildInfoRepository(buildName), encodeForBuildInfoRepository(buildNumber), timestamp, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1", "actual_md5"})) +} + +func encodeForBuildInfoRepository(value string) string { + results := "" + for _, char := range value { + if unicode.IsSpace(char) { + char = ' ' + } + if encoding, exist := specialAqlCharacters[char]; exist { + results += encoding + } else { + results += string(char) + } + } + slashEncoding := specialAqlCharacters['/'] + results = strings.ReplaceAll(results, slashEncoding+" ", slashEncoding+spaceEncoding) + results = strings.ReplaceAll(results, " "+slashEncoding, spaceEncoding+slashEncoding) + return results +} + func CreateAqlQueryForLatestCreated(repo, path string) string { itemsPart := `items.find({` + diff --git a/artifactory/services/utils/aqlquerybuilder_test.go b/artifactory/services/utils/aqlquerybuilder_test.go index 8429125ca..cd108abc2 100644 --- a/artifactory/services/utils/aqlquerybuilder_test.go +++ b/artifactory/services/utils/aqlquerybuilder_test.go @@ -220,3 +220,36 @@ func TestBuildKeyValQueryPart(t *testing.T) { }) } } + +var encodeForBuildInfoRepositoryProvider = []struct { + value string + expectedEncoding string +}{ + // Shouldn't encode + {"", ""}, + {"a", "a"}, + {"a b", "a b"}, + {"a.b", "a.b"}, + {"a&b", "a&b"}, + + // Should encode + {"a/b", "a%2Fb"}, + {"a\\b", "a%5Cb"}, + {"a:b", "a%3Ab"}, + {"a|b", "a%7Cb"}, + {"a*b", "a%2Ab"}, + {"a?b", "a%3Fb"}, + {"a / b", "a %20%2F%20 b"}, + + // Should convert whitespace to space + {"a\tb", "a b"}, + {"a\nb", "a b"}, +} + +func TestEncodeForBuildInfoRepository(t *testing.T) { + for _, testCase := range encodeForBuildInfoRepositoryProvider { + t.Run(testCase.value, func(t *testing.T) { + assert.Equal(t, testCase.expectedEncoding, encodeForBuildInfoRepository(testCase.value)) + }) + } +} diff --git a/tests/artifactoryaql_test.go b/tests/artifactoryaql_test.go new file mode 100644 index 000000000..fe4b1269f --- /dev/null +++ b/tests/artifactoryaql_test.go @@ -0,0 +1,45 @@ +package tests + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "testing" + + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/stretchr/testify/assert" +) + +func TestCreateAqlQueryForBuildInfoJson(t *testing.T) { + initArtifactoryTest(t) + + // Create a build + buildName := fmt.Sprintf("%s-%s", "a / \\ | \t * ? : ; \\ / %b", getRunId()) + err := createDummyBuild(buildName) + assert.NoError(t, err) + + // Run AQL to get the build from Artifactory + aqlQuery := utils.CreateAqlQueryForBuildInfoJson("", buildName, buildNumber, buildTimestamp) + stream, err := testsAqlService.ExecAql(aqlQuery) + assert.NoError(t, err) + defer func() { + assert.NoError(t, stream.Close()) + }() + + // Parse AQL results + aqlResults, err := io.ReadAll(stream) + assert.NoError(t, err) + parsedResult := new(utils.AqlSearchResult) + err = json.Unmarshal(aqlResults, parsedResult) + assert.NoError(t, err) + assert.Len(t, parsedResult.Results, 1) + + // Verify build checksum exist + assert.NotEmpty(t, parsedResult.Results[0].Actual_Sha1) + assert.NotEmpty(t, parsedResult.Results[0].Actual_Md5) + + // Delete build + encodedBuildName := strings.TrimSuffix(parsedResult.Results[0].Path, "-"+buildTimestamp+".json") + assert.NoError(t, deleteBuild(encodedBuildName)) +} diff --git a/tests/jfrogclient_test.go b/tests/jfrogclient_test.go index f48602460..cdba5cc94 100644 --- a/tests/jfrogclient_test.go +++ b/tests/jfrogclient_test.go @@ -55,6 +55,7 @@ func setupIntegrationTests() { createArtifactoryFederationManager() createArtifactorySystemManager() createArtifactoryStorageManager() + createArtifactoryAqlManager() } if *TestDistribution { diff --git a/tests/utils_test.go b/tests/utils_test.go index 7bd12c275..a4694afc2 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -95,6 +96,7 @@ var ( testsFederationService *services.FederationService testsSystemService *services.SystemService testsStorageService *services.StorageService + testsAqlService *services.AqlService // Distribution services testsBundleSetSigningKeyService *distributionServices.SetSigningKeyService @@ -133,6 +135,8 @@ var ( const ( HttpClientCreationFailureMessage = "Failed while attempting to create HttpClient: %s" + buildNumber = "1.0.0" + buildTimestamp = "1412067619893" ) func init() { @@ -413,6 +417,13 @@ func createArtifactoryStorageManager() { testsStorageService = services.NewStorageService(artDetails, client) } +func createArtifactoryAqlManager() { + artDetails := GetRtDetails() + client, err := createJfrogHttpClient(&artDetails) + failOnHttpClientCreation(err) + testsAqlService = services.NewAqlService(artDetails, client) +} + func createJfrogHttpClient(artDetailsPtr *auth.ServiceDetails) (*jfroghttpclient.JfrogHttpClient, error) { artDetails := *artDetailsPtr return jfroghttpclient.JfrogClientBuilder(). @@ -938,7 +949,7 @@ func isRepoExists(t *testing.T, repoKey string) bool { func createDummyBuild(buildName string) error { dataArtifactoryBuild := &buildinfo.BuildInfo{ Name: buildName, - Number: "1.0.0", + Number: buildNumber, Started: "2014-09-30T12:00:19.893+0300", Modules: []buildinfo.Module{{ Id: "example-module", @@ -959,11 +970,6 @@ func createDummyBuild(buildName string) error { } func deleteBuild(buildName string) error { - err := deleteBuildIndex(buildName) - if err != nil { - return err - } - artDetails := GetRtDetails() artHTTPDetails := artDetails.CreateHttpClientDetails() client, err := httpclient.ClientBuilder().Build() @@ -971,12 +977,13 @@ func deleteBuild(buildName string) error { return err } - resp, _, err := client.SendDelete(artDetails.GetUrl()+"api/build/"+buildName+"?deleteAll=1", nil, artHTTPDetails, "") + buildName = url.PathEscape(buildName) + resp, _, err := client.SendDelete(artDetails.GetUrl()+"artifactory-build-info/"+buildName, nil, artHTTPDetails, "") if err != nil { return err } - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusNoContent { return errors.New("failed to delete build " + resp.Status) } diff --git a/tests/xraybinmgr_test.go b/tests/xraybinmgr_test.go index 278b4bdc8..1a645b783 100644 --- a/tests/xraybinmgr_test.go +++ b/tests/xraybinmgr_test.go @@ -15,6 +15,7 @@ func TestXrayBinMgr(t *testing.T) { func addBuildsToIndexing(t *testing.T) { buildName := fmt.Sprintf("%s-%s", "build1", getRunId()) defer func() { + assert.NoError(t, deleteBuildIndex(buildName)) assert.NoError(t, deleteBuild(buildName)) }() // Create a build diff --git a/tests/xraywatch_test.go b/tests/xraywatch_test.go index d7b2ca2e3..30d0f4e0c 100644 --- a/tests/xraywatch_test.go +++ b/tests/xraywatch_test.go @@ -133,12 +133,14 @@ func testXrayWatchSelectedRepos(t *testing.T) { err = createAndIndexBuild(t, build1Name) assert.NoError(t, err) defer func() { + assert.NoError(t, deleteBuildIndex(build1Name)) assert.NoError(t, deleteBuild(build1Name)) }() build2Name := fmt.Sprintf("%s-%s", "build2", getRunId()) err = createAndIndexBuild(t, build2Name) assert.NoError(t, err) defer func() { + assert.NoError(t, deleteBuildIndex(build2Name)) assert.NoError(t, deleteBuild(build2Name)) }() paramsSelectedRepos := utils.NewWatchParams() From 93c64fa231c78c04fbbc16914352b7d5ccd42ac7 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Thu, 9 Nov 2023 17:54:05 +0200 Subject: [PATCH 3/3] Promoted version to 1.34.5 (#859) --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index 886a2e9c5..6640c04c1 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -27,7 +27,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.34.4" + Version = "1.34.5" ) type MinVersionProduct string