From 6f4d83a2c564d278c0e838df0e4f62f52c15b7df Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 11:46:17 +0300 Subject: [PATCH 01/26] copy dir Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index 5aa3dfad..40498d66 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -3,6 +3,7 @@ package utils import ( "bufio" "encoding/json" + "errors" "fmt" "golang.org/x/exp/slices" "io" @@ -415,15 +416,15 @@ func CopyDir(fromPath, toPath string, includeDirs bool, excludeNames []string) e if slices.Contains(excludeNames, filepath.Base(v)) { continue } - - dir, err := IsDirExists(v, false) + var exists bool + exists, err = IsDirExists(v, false) if err != nil { return err } - if dir { + if exists { toPath := toPath + GetFileSeparator() + filepath.Base(v) - err := CopyDir(v, toPath, true, nil) + err = CopyDir(v, toPath, true, nil) if err != nil { return err } @@ -440,31 +441,29 @@ func CopyDir(fromPath, toPath string, includeDirs bool, excludeNames []string) e func CopyFile(dst, src string) (err error) { srcFile, err := os.Open(src) if err != nil { - return err + return } defer func() { - e := srcFile.Close() - if err == nil { - err = e - } + err = errors.Join(err, srcFile.Close()) }() + srcInfo, err := srcFile.Stat() + if err != nil { + return + } fileName, _ := GetFileAndDirFromPath(src) dstPath, err := CreateFilePath(dst, fileName) if err != nil { - return err + return } - dstFile, err := os.Create(dstPath) + dstFile, err := os.OpenFile(dstPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, srcInfo.Mode()) if err != nil { - return err + return } defer func() { - e := dstFile.Close() - if err == nil { - err = e - } + err = errors.Join(err, dstFile.Close()) }() _, err = io.Copy(dstFile, srcFile) - return err + return } func GetFileSeparator() string { From 99ea1c7c43804838645760071bb278684b6ea68d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 11:48:58 +0300 Subject: [PATCH 02/26] copy dir Signed-off-by: Michael Sverdlov --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4848b166..46df39a6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/BurntSushi/toml v1.3.2 - github.com/CycloneDX/cyclonedx-go v0.7.1 + github.com/CycloneDX/cyclonedx-go v0.7.2 github.com/buger/jsonparser v1.1.1 github.com/jfrog/gofrog v1.3.0 github.com/minio/sha256-simd v1.0.1 diff --git a/go.sum b/go.sum index 0f1e7399..427c2c90 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/CycloneDX/cyclonedx-go v0.7.1 h1:5w1SxjGm9MTMNTuRbEPyw21ObdbaagTWF/KfF0qHTRE= -github.com/CycloneDX/cyclonedx-go v0.7.1/go.mod h1:N/nrdWQI2SIjaACyyDs/u7+ddCkyl/zkNs8xFsHF2Ps= +github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= +github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -24,6 +24,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= From a2be3c3ed7631b6ed6a94d94ef7fbb7a941570ba Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 11:56:23 +0300 Subject: [PATCH 03/26] copy dir Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index 40498d66..9d0cfced 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -416,13 +416,13 @@ func CopyDir(fromPath, toPath string, includeDirs bool, excludeNames []string) e if slices.Contains(excludeNames, filepath.Base(v)) { continue } - var exists bool - exists, err = IsDirExists(v, false) + var isDir bool + isDir, err = IsDirExists(v, false) if err != nil { return err } - if exists { + if isDir { toPath := toPath + GetFileSeparator() + filepath.Base(v) err = CopyDir(v, toPath, true, nil) if err != nil { From 15c2382ac94b1310955192e279bf99c03eec58aa Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 12:05:34 +0300 Subject: [PATCH 04/26] copy dir Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index 9d0cfced..921cb54a 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -411,31 +411,28 @@ func CopyDir(fromPath, toPath string, includeDirs bool, excludeNames []string) e return err } - for _, v := range files { + for _, file := range files { + fileName := filepath.Base(file) // Skip if excluded - if slices.Contains(excludeNames, filepath.Base(v)) { + if slices.Contains(excludeNames, fileName) { continue } var isDir bool - isDir, err = IsDirExists(v, false) + isDir, err = IsDirExists(file, false) if err != nil { return err } if isDir { - toPath := toPath + GetFileSeparator() + filepath.Base(v) - err = CopyDir(v, toPath, true, nil) - if err != nil { - return err - } - continue + err = CopyDir(file, filepath.Join(toPath, fileName), true, nil) + } else { + err = CopyFile(toPath, file) } - err = CopyFile(toPath, v) if err != nil { return err } } - return err + return nil } func CopyFile(dst, src string) (err error) { From d1ba55319de3bb6c4140ec1903b743e6b5ea9166 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 17:58:05 +0300 Subject: [PATCH 05/26] copy dir Signed-off-by: Michael Sverdlov --- build/utils/npm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index 114783ef..2d43e1e2 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -403,7 +403,7 @@ func GetNpmVersionAndExecPath(log utils.Log) (*version.Version, string, error) { if err != nil { return nil, "", err } - return version.NewVersion(string(versionData)), npmExecPath, nil + return version.NewVersion(strings.TrimSpace(string(versionData))), npmExecPath, nil } type PackageInfo struct { From 43f807c3438008ee74487f3545267b5e7a15f1c4 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 18:27:35 +0300 Subject: [PATCH 06/26] copy dir Signed-off-by: Michael Sverdlov --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a735896f..db6fe501 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu, windows, macOS ] + os: [ ubuntu, windows, macos ] node: [ 14, 16, 16.9 ] include: - node: "14" From 71df92d19b5ffa599fc8e5b5942ac6c2a3b95026 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Sep 2023 21:34:29 +0300 Subject: [PATCH 07/26] fix build script Signed-off-by: Michael Sverdlov --- go.mod | 4 ++-- go.sum | 8 ++++---- release/pipelines.yml | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 46df39a6..f13c6311 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.7 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( @@ -23,6 +23,6 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/sys v0.11.0 // indirect + golang.org/x/sys v0.12.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 427c2c90..66abff06 100644 --- a/go.sum +++ b/go.sum @@ -35,11 +35,11 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/release/pipelines.yml b/release/pipelines.yml index 02d5fad6..4148d55c 100644 --- a/release/pipelines.yml +++ b/release/pipelines.yml @@ -69,7 +69,6 @@ pipelines: - jf rt bp $JFROG_CLI_BUILD_NAME $JFROG_CLI_BUILD_NUMBER # Update build-info.json - - curl -X GET "https://releases.jfrog.io/artifactory/bi-cli/v1/%5BRELEASE%5D/$operating_system-$architecture/bi" -L -o bi - chmod u+x bi - ./bi go > build-info.json - git add build-info.json From 417516b075d34241fc24f503a92b86897cfad051 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Sep 2023 21:46:22 +0300 Subject: [PATCH 08/26] Update release/pipelines.yml Co-authored-by: Yahav Itzhak --- release/pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/release/pipelines.yml b/release/pipelines.yml index 4148d55c..5addfaff 100644 --- a/release/pipelines.yml +++ b/release/pipelines.yml @@ -69,6 +69,7 @@ pipelines: - jf rt bp $JFROG_CLI_BUILD_NAME $JFROG_CLI_BUILD_NUMBER # Update build-info.json + - ./buildscripts/build.sh - chmod u+x bi - ./bi go > build-info.json - git add build-info.json From d37e048217da4c7a8692145ac29ef1e2d879cdef Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Sep 2023 21:48:24 +0300 Subject: [PATCH 09/26] fix build script Signed-off-by: Michael Sverdlov --- release/pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/pipelines.yml b/release/pipelines.yml index 4148d55c..8996aff9 100644 --- a/release/pipelines.yml +++ b/release/pipelines.yml @@ -64,7 +64,7 @@ pipelines: JFROG_CLI_BUILD_NAME=$JFROG_CLI_BUILD_NAME JFROG_CLI_BUILD_NUMBER=$JFROG_CLI_BUILD_NUMBER JFROG_CLI_BUILD_PROJECT=$JFROG_CLI_BUILD_PROJECT - release/build.sh "$NEXT_VERSION" + buildscripts/build.sh "$NEXT_VERSION" - jf rt bag && jf rt bce - jf rt bp $JFROG_CLI_BUILD_NAME $JFROG_CLI_BUILD_NUMBER From d15dfa75075c0f883f7b393e92e27e43bae56d07 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Sep 2023 21:51:59 +0300 Subject: [PATCH 10/26] fix build script Signed-off-by: Michael Sverdlov --- release/pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/pipelines.yml b/release/pipelines.yml index 086d8332..5addfaff 100644 --- a/release/pipelines.yml +++ b/release/pipelines.yml @@ -64,7 +64,7 @@ pipelines: JFROG_CLI_BUILD_NAME=$JFROG_CLI_BUILD_NAME JFROG_CLI_BUILD_NUMBER=$JFROG_CLI_BUILD_NUMBER JFROG_CLI_BUILD_PROJECT=$JFROG_CLI_BUILD_PROJECT - buildscripts/build.sh "$NEXT_VERSION" + release/build.sh "$NEXT_VERSION" - jf rt bag && jf rt bce - jf rt bp $JFROG_CLI_BUILD_NAME $JFROG_CLI_BUILD_NUMBER From 3f6f620fa2c6566d334fc6d32572d14d94ce41ca Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 12 Sep 2023 20:16:27 +0300 Subject: [PATCH 11/26] fix build script Signed-off-by: Michael Sverdlov --- utils/goutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/goutils.go b/utils/goutils.go index 398f06d5..a74763ee 100644 --- a/utils/goutils.go +++ b/utils/goutils.go @@ -102,7 +102,7 @@ func GetDependenciesList(projectDir string, log Log) (map[string]bool, error) { output, err := runDependenciesCmd(projectDir, append(cmdArgs, "-f", "{{with .Module}}{{.Path}}:{{.Version}}{{end}}", "all"), log) if err != nil { // Errors occurred while running "go list". Run again and this time ignore errors (with '-e') - log.Warn("Errors occurred while building the Go dependency tree. The dependency tree may be incomplete:" + err.Error()) + log.Warn("Errors occurred while building the Go dependency tree. The dependency tree may be incomplete: " + err.Error()) output, err = runDependenciesCmd(projectDir, append(cmdArgs, "-e", "-f", "{{with .Module}}{{.Path}}:{{.Version}}{{end}}", "all"), log) if err != nil { return nil, err From 4723df341a4f0fbd0ae74304b5b5e50236c59180 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 14 Sep 2023 11:43:47 +0300 Subject: [PATCH 12/26] fix build script Signed-off-by: Michael Sverdlov --- release/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/build.sh b/release/build.sh index 73f604d5..d928a280 100755 --- a/release/build.sh +++ b/release/build.sh @@ -9,8 +9,8 @@ build () { exeName="$4" echo "Building $exeName for $GOOS-$GOARCH ..." - CGO_ENABLED=0 jf go build -o "$exeName" -ldflags '-w -extldflags "-static" -X main.cliVersion='$version main.go - chmod +x $exeName + CGO_ENABLED=0 jf go build -o "$exeName" -ldflags '-w -extldflags "-static" -X main.cliVersion='"$version" main.go + chmod +x "$exeName" # Run verification after building plugin for the correct platform of this image. if [[ "$pkg" = "linux-386" ]]; then @@ -26,7 +26,7 @@ buildAndUpload () { fileExtension="$4" exeName="bi$fileExtension" - build $pkg $goos $goarch $exeName + build "$pkg" "$goos" "$goarch" "$exeName" destPath="$pkgPath/$version/$pkg/$exeName" echo "Uploading $exeName to $destPath ..." From b9c2b05907d00d1d261caff1abc138b51165ca49 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 14 Sep 2023 11:43:51 +0300 Subject: [PATCH 13/26] fix build script Signed-off-by: Michael Sverdlov --- buildscripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/build.sh b/buildscripts/build.sh index e04255f5..5b920623 100755 --- a/buildscripts/build.sh +++ b/buildscripts/build.sh @@ -7,4 +7,4 @@ if [ $# -eq 0 ] exe_name="$1" fi -CGO_ENABLED=0 go build -o $exe_name -ldflags '-w -extldflags "-static"' main.go \ No newline at end of file +CGO_ENABLED=0 go build -o "$exe_name" -ldflags '-w -extldflags "-static"' main.go \ No newline at end of file From 6e1e42a8e005be534bd0be3aedc48015da6c5e75 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 14 Sep 2023 16:46:11 +0300 Subject: [PATCH 14/26] fix build script Signed-off-by: Michael Sverdlov --- .github/workflows/analysis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 52011c1f..802ad976 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -36,3 +36,15 @@ jobs: uses: securego/gosec@master with: args: -exclude G107,G204,G301,G302,G304,G306 -tests -exclude-dir \.*test\.* ./... + + ShellCheck: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v3 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + ignore_paths: + .husky + *test* \ No newline at end of file From 8ecbe93ac67a11077e7f82d0883af62ee89dd60d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 14 Sep 2023 16:48:28 +0300 Subject: [PATCH 15/26] fix build script Signed-off-by: Michael Sverdlov --- .github/workflows/analysis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 802ad976..5dd880af 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -45,6 +45,4 @@ jobs: - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: - ignore_paths: - .husky - *test* \ No newline at end of file + ignore_paths: \.*test\.* \ No newline at end of file From 08989d7d28627639d291e882a3efd5357056bf7a Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 14 Sep 2023 16:49:45 +0300 Subject: [PATCH 16/26] fix build script Signed-off-by: Michael Sverdlov --- .github/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 5dd880af..1a9354eb 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -45,4 +45,4 @@ jobs: - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: - ignore_paths: \.*test\.* \ No newline at end of file + ignore_paths: '*test*' \ No newline at end of file From a6875ca269850fcb38bb57f97c7bbdc5825091d5 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 20 Sep 2023 17:31:39 +0300 Subject: [PATCH 17/26] fix build script Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 96 ++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index 921cb54a..a9cb3439 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -107,50 +107,44 @@ func MoveFile(sourcePath, destPath string) (err error) { var inputFile *os.File inputFile, err = os.Open(sourcePath) if err != nil { - return err + return } defer func() { if inputFileOpen { - e := inputFile.Close() - if err == nil { - err = e - } + err = errors.Join(err, inputFile.Close()) } }() inputFileInfo, err := inputFile.Stat() if err != nil { - return err + return } var outputFile *os.File outputFile, err = os.Create(destPath) if err != nil { - return err + return } defer func() { - e := outputFile.Close() - if err == nil { - err = e - } + err = errors.Join(err, outputFile.Close()) }() _, err = io.Copy(outputFile, inputFile) if err != nil { - return err + return } err = os.Chmod(destPath, inputFileInfo.Mode()) if err != nil { - return err + return } // The copy was successful, so now delete the original file err = inputFile.Close() if err != nil { - return err + return } inputFileOpen = false err = os.Remove(sourcePath) - return err + return } // Return the list of files and directories in the specified path @@ -232,12 +226,10 @@ func DownloadFile(downloadTo string, fromUrl string) (err error) { // Get the data resp, err := http.Get(fromUrl) if err != nil { - return err + return } defer func() { - if deferErr := resp.Body.Close(); err == nil { - err = deferErr - } + err = errors.Join(err, resp.Body.Close()) }() if resp.StatusCode != http.StatusOK { err = fmt.Errorf("download failed. status code: %s", resp.Status) @@ -250,9 +242,7 @@ func DownloadFile(downloadTo string, fromUrl string) (err error) { return } defer func() { - if deferErr := out.Close(); err == nil { - err = deferErr - } + err = errors.Join(err, out.Close()) }() // Write the body to file _, err = io.Copy(out, resp.Body) @@ -311,22 +301,19 @@ func removeDirContents(dirPath string) (err error) { return } defer func() { - e := d.Close() - if err == nil { - err = e - } + err = errors.Join(err, d.Close()) }() names, err := d.Readdirnames(-1) if err != nil { - return err + return } for _, name := range names { err = os.RemoveAll(filepath.Join(dirPath, name)) if err != nil { - return err + return } } - return nil + return } // Old runs/tests may leave junk at temp dir. @@ -535,17 +522,15 @@ func Unmarshal(filePath string, loadTarget interface{}) (err error) { return } defer func() { - closeErr := jsonFile.Close() - if err == nil { - err = closeErr - } + err = errors.Join(err, jsonFile.Close()) }() var byteValue []byte byteValue, err = io.ReadAll(jsonFile) if err != nil { return } - return json.Unmarshal(byteValue, &loadTarget) + err = json.Unmarshal(byteValue, &loadTarget) + return } // strip '\n' or read until EOF, return error if read error @@ -553,26 +538,25 @@ func Unmarshal(filePath string, loadTarget interface{}) (err error) { func ReadNLines(path string, total int) (lines []string, err error) { reader, err := os.Open(path) if err != nil { - return nil, err + return } defer func() { - deferErr := reader.Close() - if err == nil { - err = deferErr - } + err = errors.Join(err, reader.Close()) }() r := bufio.NewReader(reader) for i := 0; i < total; i++ { - line, _, err := r.ReadLine() + var line []byte + line, _, err = r.ReadLine() lines = append(lines, string(line)) if err == io.EOF { + err = nil break } if err != nil { - return nil, err + return } } - return lines, nil + return } type FileDetails struct { @@ -585,7 +569,7 @@ func GetFileDetails(filePath string, includeChecksums bool) (details *FileDetail if includeChecksums { details.Checksum, err = calcChecksumDetails(filePath) if err != nil { - return details, err + return } } else { details.Checksum = entities.Checksum{} @@ -593,34 +577,30 @@ func GetFileDetails(filePath string, includeChecksums bool) (details *FileDetail file, err := os.Open(filePath) defer func() { - e := file.Close() - if err == nil { - err = e - } + err = errors.Join(err, file.Close()) }() if err != nil { - return nil, err + return } fileInfo, err := file.Stat() if err != nil { - return nil, err + return } details.Size = fileInfo.Size() - return details, nil + return } func calcChecksumDetails(filePath string) (checksum entities.Checksum, err error) { file, err := os.Open(filePath) - defer func() { - e := file.Close() - if err == nil { - err = e - } - }() if err != nil { - return entities.Checksum{}, err + return } - return calcChecksumDetailsFromReader(file) + defer func() { + err = errors.Join(err, file.Close()) + }() + + checksum, err = calcChecksumDetailsFromReader(file) + return } func calcChecksumDetailsFromReader(reader io.Reader) (entities.Checksum, error) { From cf1f3c5ed8ac5e8ce0015c04739ffb5d612de7cb Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 26 Sep 2023 12:09:40 +0300 Subject: [PATCH 18/26] Update utils/fileutils.go Co-authored-by: Yahav Itzhak --- utils/fileutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index a9cb3439..4cafb76d 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -543,7 +543,7 @@ func ReadNLines(path string, total int) (lines []string, err error) { defer func() { err = errors.Join(err, reader.Close()) }() - r := bufio.NewReader(reader) + bufferedReader := bufio.NewReader(reader) for i := 0; i < total; i++ { var line []byte line, _, err = r.ReadLine() From b280efbae5a619805b8c7ef645dca032ee865a55 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 26 Sep 2023 12:12:53 +0300 Subject: [PATCH 19/26] fix build script Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index a9cb3439..5c6e19a8 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -599,14 +599,10 @@ func calcChecksumDetails(filePath string) (checksum entities.Checksum, err error err = errors.Join(err, file.Close()) }() - checksum, err = calcChecksumDetailsFromReader(file) - return -} - -func calcChecksumDetailsFromReader(reader io.Reader) (entities.Checksum, error) { - checksumInfo, err := CalcChecksums(reader) + checksumInfo, err := CalcChecksums(file) if err != nil { return entities.Checksum{}, err } - return entities.Checksum{Md5: checksumInfo[MD5], Sha1: checksumInfo[SHA1], Sha256: checksumInfo[SHA256]}, nil + checksum = entities.Checksum{Md5: checksumInfo[MD5], Sha1: checksumInfo[SHA1], Sha256: checksumInfo[SHA256]} + return } From c94d4726f6ca0015977fcc11f0c0ca0324ac22f9 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 26 Sep 2023 12:13:21 +0300 Subject: [PATCH 20/26] fix build script Signed-off-by: Michael Sverdlov --- utils/fileutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/fileutils.go b/utils/fileutils.go index f58ad43b..7bc59b04 100644 --- a/utils/fileutils.go +++ b/utils/fileutils.go @@ -546,7 +546,7 @@ func ReadNLines(path string, total int) (lines []string, err error) { bufferedReader := bufio.NewReader(reader) for i := 0; i < total; i++ { var line []byte - line, _, err = r.ReadLine() + line, _, err = bufferedReader.ReadLine() lines = append(lines, string(line)) if err == io.EOF { err = nil From 74d3cd1bea39c5d860ebc4caf23fb5f558470535 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Sep 2023 19:26:37 +0300 Subject: [PATCH 21/26] fix build script Signed-off-by: Michael Sverdlov --- build/utils/npm.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index 2d43e1e2..f841488b 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -351,14 +351,13 @@ func appendScopes(oldScopes []string, newScopes []string) []string { } func RunNpmCmd(executablePath, srcPath string, npmArgs []string, log utils.Log) (stdResult, errResult []byte, err error) { - log.Debug("Running npm " + npmArgs[0] + " command.") args := make([]string, 0) for i := 0; i < len(npmArgs); i++ { if strings.TrimSpace(npmArgs[i]) != "" { args = append(args, npmArgs[i]) } } - + log.Debug("Running 'npm " + strings.Join(npmArgs, " ") + "' command.") command := exec.Command(executablePath, args...) command.Dir = srcPath outBuffer := bytes.NewBuffer([]byte{}) @@ -369,7 +368,7 @@ func RunNpmCmd(executablePath, srcPath string, npmArgs []string, log utils.Log) errResult = errBuffer.Bytes() stdResult = outBuffer.Bytes() if err != nil { - err = fmt.Errorf("error while running '%s %s': %s\n%s", executablePath, strings.Join(args, " "), strings.TrimSpace(string(errResult)), err.Error()) + err = fmt.Errorf("error while running '%s %s': %s\n%s", executablePath, strings.Join(args, " "), err.Error(), strings.TrimSpace(string(errResult))) return } log.Debug("npm '" + strings.Join(args, " ") + "' standard output is:\n" + strings.TrimSpace(string(stdResult))) From b47492209bd5cb9b5eb79785645f969cb79a868d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Dec 2023 15:59:41 +0200 Subject: [PATCH 22/26] fix latest build Signed-off-by: Michael Sverdlov --- build/utils/dotnet/dependencies/packagesconfig.go | 6 ++++-- .../packagesproject/localcache/id2/2.0.0/id2.nuspec | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index 372a0c31..d784de84 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -218,9 +218,11 @@ func createNugetPackage(packagesPath string, nuget xmlPackage, nPackage *nugetPa if err != nil { return nil, err } - + // Sometimes the nuspec file is encoded in utf-16. + // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert it to utf-8. + utf8String := strings.Replace(string(nuspecContent), "utf-16", "utf-8", 1) nuspec := &nuspec{} - err = xml.Unmarshal(nuspecContent, nuspec) + err = xml.Unmarshal([]byte(utf8String), nuspec) if err != nil { pack := nPackage.id + ":" + nPackage.version log.Warn("Package:", pack, "couldn't be parsed due to:", err.Error(), ". Skipping the package dependency.") diff --git a/build/utils/dotnet/dependencies/testdata/packagesproject/localcache/id2/2.0.0/id2.nuspec b/build/utils/dotnet/dependencies/testdata/packagesproject/localcache/id2/2.0.0/id2.nuspec index 3fbcf4df..f3c2ab65 100644 --- a/build/utils/dotnet/dependencies/testdata/packagesproject/localcache/id2/2.0.0/id2.nuspec +++ b/build/utils/dotnet/dependencies/testdata/packagesproject/localcache/id2/2.0.0/id2.nuspec @@ -1,4 +1,4 @@ - + id2 From 5acf77b0c4d55690198708b3235e700cc2b07306 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Dec 2023 16:11:42 +0200 Subject: [PATCH 23/26] fix latest build Signed-off-by: Michael Sverdlov --- build/utils/dotnet/dependencies/packagesconfig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index d784de84..39499ead 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -218,8 +218,8 @@ func createNugetPackage(packagesPath string, nuget xmlPackage, nPackage *nugetPa if err != nil { return nil, err } - // Sometimes the nuspec file is encoded in utf-16. - // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert it to utf-8. + // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'enocding="utf-16"' key. + // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert the header to utf-8. utf8String := strings.Replace(string(nuspecContent), "utf-16", "utf-8", 1) nuspec := &nuspec{} err = xml.Unmarshal([]byte(utf8String), nuspec) From c3e3e94c203fbc393f0cb470d2d7c9e09a31dc08 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Dec 2023 16:25:16 +0200 Subject: [PATCH 24/26] fix latest build Signed-off-by: Michael Sverdlov --- .../dotnet/dependencies/packagesconfig.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index 39499ead..db864d7a 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -133,7 +133,7 @@ func (extractor *packagesExtractor) loadPackagesConfig(dependenciesSource string } config := &packagesConfig{} - err = xml.Unmarshal(content, config) + err = xmlUnmarshal(content, config) if err != nil { return nil, err } @@ -218,11 +218,8 @@ func createNugetPackage(packagesPath string, nuget xmlPackage, nPackage *nugetPa if err != nil { return nil, err } - // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'enocding="utf-16"' key. - // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert the header to utf-8. - utf8String := strings.Replace(string(nuspecContent), "utf-16", "utf-8", 1) nuspec := &nuspec{} - err = xml.Unmarshal([]byte(utf8String), nuspec) + err = xmlUnmarshal(nuspecContent, nuspec) if err != nil { pack := nPackage.id + ":" + nPackage.version log.Warn("Package:", pack, "couldn't be parsed due to:", err.Error(), ". Skipping the package dependency.") @@ -312,3 +309,15 @@ type group struct { TargetFramework string `xml:"targetFramework,attr"` Dependencies []xmlPackage `xml:"dependency"` } + +// xmlUnmarshal is a wrapper for xml.Unmarshal, handling wrongly encoded utf-16 XML by replacing "utf-16" with "utf-8" in the header. +func xmlUnmarshal(content []byte, obj interface{}) (err error) { + err = xml.Unmarshal(content, obj) + if err != nil { + // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'enocding="utf-16"' key. + // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert the header to utf-8. + utf8String := strings.Replace(string(content), "utf-16", "utf-8", 1) + err = xml.Unmarshal([]byte(utf8String), obj) + } + return +} From ba9935cec2250b2c0c4c2841332dfea80d803254 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Dec 2023 16:29:32 +0200 Subject: [PATCH 25/26] fix latest build Signed-off-by: Michael Sverdlov --- build/utils/dotnet/dependencies/packagesconfig.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index db864d7a..d752801c 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -1,6 +1,7 @@ package dependencies import ( + "bytes" "encoding/xml" "fmt" "github.com/jfrog/build-info-go/utils" @@ -316,8 +317,8 @@ func xmlUnmarshal(content []byte, obj interface{}) (err error) { if err != nil { // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'enocding="utf-16"' key. // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert the header to utf-8. - utf8String := strings.Replace(string(content), "utf-16", "utf-8", 1) - err = xml.Unmarshal([]byte(utf8String), obj) + utf8Bytes := bytes.Replace(content, []byte("utf-16"), []byte("utf-8"), 1) + err = xml.Unmarshal(utf8Bytes, obj) } return } From 5e7fd1ced2a3a301b32a58f69988346bb7dac196 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 27 Dec 2023 16:39:13 +0200 Subject: [PATCH 26/26] fix latest build Signed-off-by: Michael Sverdlov --- build/utils/dotnet/dependencies/packagesconfig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/utils/dotnet/dependencies/packagesconfig.go b/build/utils/dotnet/dependencies/packagesconfig.go index d752801c..2647b2db 100644 --- a/build/utils/dotnet/dependencies/packagesconfig.go +++ b/build/utils/dotnet/dependencies/packagesconfig.go @@ -74,9 +74,9 @@ func (extractor *packagesExtractor) extract(packagesConfig *packagesConfig, glob return err } if pack == nil { - // If doesn't exists lets build the array of alternative versions. + // If it doesn't exist lets build the array of alternative versions. alternativeVersions := createAlternativeVersionForms(nuget.Version) - // Now lets do a loop to run over the alternative possibilities + // Now let's do a loop to run over the alternative possibilities for i := 0; i < len(alternativeVersions); i++ { nPackage.version = alternativeVersions[i] pack, err = createNugetPackage(globalPackagesCache, nuget, nPackage, log) @@ -315,7 +315,7 @@ type group struct { func xmlUnmarshal(content []byte, obj interface{}) (err error) { err = xml.Unmarshal(content, obj) if err != nil { - // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'enocding="utf-16"' key. + // Sometimes the nuspec file is wrongly encoded in utf-16, the actual encoding is utf-8 but the xml header has 'encoding="utf-16"' key. // xml.Unmarshal doesn't support utf-16 encoding, so we need to convert the header to utf-8. utf8Bytes := bytes.Replace(content, []byte("utf-16"), []byte("utf-8"), 1) err = xml.Unmarshal(utf8Bytes, obj)