diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 747cbdd41..1cd20256a 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -36,7 +36,8 @@ jobs: with: go-version: 1.20.x + # Temporarily set version 2.18.0 to workaround https://github.com/securego/gosec/issues/1046 - name: Run Gosec Security Scanner - uses: securego/gosec@master + uses: securego/gosec@v2.18.0 with: args: -exclude G204,G301,G302,G304,G306 -tests -exclude-dir \.*test\.* ./... diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 31219f00e..0292e51fc 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -174,6 +174,13 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon return err } + if base.progressBar != nil { + base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts)) + } + if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil { + log.Warn("Couldn't decrease the delayed files counter", err.Error()) + } + // Remove the file, so it won't be consumed again. if err = os.Remove(filePath); err != nil { return errorutils.CheckError(err) @@ -203,6 +210,23 @@ func getDelayFiles(repoKeys []string) (filesPaths []string, err error) { return getErrorOrDelayFiles(repoKeys, getJfrogTransferRepoDelaysDir) } +func getDelayedFilesCount(repoKeys []string) (int, error) { + files, err := getDelayFiles(repoKeys) + if err != nil { + return -1, err + } + + count := 0 + for _, file := range files { + delayedFiles, err := readDelayFile(file) + if err != nil { + return -1, err + } + count += len(delayedFiles.DelayedArtifacts) + } + return count, nil +} + const ( maven = "Maven" gradle = "Gradle" @@ -256,6 +280,12 @@ func (delayHelper delayUploadHelper) delayUploadIfNecessary(phase phaseBase, fil if shouldDelay(file.Name) { delayed = true delayHelper.delayedArtifactsChannelMng.add(file) + if phase.progressBar != nil { + phase.progressBar.changeNumberOfDelayedFiles(1) + } + if err := phase.stateManager.ChangeDelayedFilesCountBy(1, true); err != nil { + log.Warn("Couldn't increase the delayed files counter", err.Error()) + } } } return diff --git a/artifactory/commands/transferfiles/state/runstatus.go b/artifactory/commands/transferfiles/state/runstatus.go index 1b9ffa9d6..d9d48e032 100644 --- a/artifactory/commands/transferfiles/state/runstatus.go +++ b/artifactory/commands/transferfiles/state/runstatus.go @@ -36,6 +36,7 @@ type TransferRunStatus struct { BuildInfoRepo bool `json:"build_info_repo,omitempty"` CurrentRepoPhase int `json:"current_repo_phase,omitempty"` WorkingThreads int `json:"working_threads,omitempty"` + DelayedFiles uint `json:"delayed_files,omitempty"` TransferFailures uint `json:"transfer_failures,omitempty"` TimeEstimationManager `json:"time_estimation,omitempty"` StaleChunks []StaleChunks `json:"stale_chunks,omitempty"` diff --git a/artifactory/commands/transferfiles/state/statemanager.go b/artifactory/commands/transferfiles/state/statemanager.go index 5425832e1..82adbc92e 100644 --- a/artifactory/commands/transferfiles/state/statemanager.go +++ b/artifactory/commands/transferfiles/state/statemanager.go @@ -263,6 +263,17 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er }) } +func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error { + return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { + if increase { + transferRunStatus.DelayedFiles += count + } else { + transferRunStatus.DelayedFiles -= count + } + return nil + }) +} + func (ts *TransferStateManager) ChangeTransferFailureCountBy(count uint, increase bool) error { return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error { if increase { diff --git a/artifactory/commands/transferfiles/status.go b/artifactory/commands/transferfiles/status.go index db95454cd..818c3db55 100644 --- a/artifactory/commands/transferfiles/status.go +++ b/artifactory/commands/transferfiles/status.go @@ -10,6 +10,7 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-core/v2/utils/progressbar" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -78,9 +79,14 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings. addString(output, "๐Ÿงต", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2) addString(output, "โšก", "Transfer speed", stateManager.GetSpeedString(), 2) addString(output, "โŒ›", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1) + delayedTxt := strconv.FormatUint(uint64(stateManager.DelayedFiles), 10) + if stateManager.DelayedFiles > 0 { + delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")" + } + addString(output, "โœ‹", "Delayed files", delayedTxt, 2) failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10) if stateManager.TransferFailures > 0 { - failureTxt += " (" + "In Phase 3 and in subsequent executions, we'll retry transferring the failed files." + ")" + failureTxt += " (" + progressbar.RetryFailureContentNote + ")" } addString(output, "โŒ", "Transfer failures", failureTxt, 2) } diff --git a/artifactory/commands/transferfiles/status_test.go b/artifactory/commands/transferfiles/status_test.go index f1c7226ab..995f94ad8 100644 --- a/artifactory/commands/transferfiles/status_test.go +++ b/artifactory/commands/transferfiles/status_test.go @@ -69,7 +69,8 @@ func TestShowStatus(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") assert.Contains(t, results, "Estimated time remaining: Less than a minute") - assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files.)") + assert.Contains(t, results, "Delayed files: 20 (Files to be transferred last, after all other files)") + assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)") // Check repository status assert.Contains(t, results, "Current Repository Status") @@ -99,7 +100,8 @@ func TestShowStatusDiffPhase(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: 0.011 MB/s") assert.Contains(t, results, "Estimated time remaining: Not available in this phase") - assert.Contains(t, results, "Transfer failures: 223") + assert.Contains(t, results, "Delayed files: 20 (Files to be transferred last, after all other files)") + assert.Contains(t, results, "Transfer failures: 223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)") // Check repository status assert.Contains(t, results, "Current Repository Status") @@ -129,6 +131,7 @@ func TestShowBuildInfoRepo(t *testing.T) { assert.Contains(t, results, "Working threads: 16") assert.Contains(t, results, "Transfer speed: Not available while transferring a build-info repository") assert.Contains(t, results, "Estimated time remaining: Less than a minute") + assert.Contains(t, results, "Delayed files: 20 (Files to be transferred last, after all other files)") assert.Contains(t, results, "Transfer failures: 223") // Check repository status @@ -174,11 +177,12 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks stateManager.TotalRepositories.TotalUnits = 1111 stateManager.TotalRepositories.TransferredUnits = 15 stateManager.WorkingThreads = 16 + stateManager.DelayedFiles = 20 stateManager.TransferFailures = 223 - stateManager.TimeEstimationManager.LastSpeeds = []float64{12} - stateManager.TimeEstimationManager.LastSpeedsSum = 12 - stateManager.TimeEstimationManager.SpeedsAverage = 12 + stateManager.LastSpeeds = []float64{12} + stateManager.LastSpeedsSum = 12 + stateManager.SpeedsAverage = 12 if staleChunks { stateManager.StaleChunks = append(stateManager.StaleChunks, state.StaleChunks{ diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index 218a9fb28..40faa5e7c 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -228,8 +228,15 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui return e } tdc.stateManager.TransferFailures = uint(numberInitialErrors) + + numberInitialDelays, e := getDelayedFilesCount(allSourceLocalRepos) + if e != nil { + return e + } + tdc.stateManager.DelayedFiles = uint(numberInitialDelays) } else { tdc.stateManager.TransferFailures = 0 + tdc.stateManager.DelayedFiles = 0 } return nil } diff --git a/artifactory/commands/transferfiles/transferfileprogress.go b/artifactory/commands/transferfiles/transferfileprogress.go index 4f26b2bc0..059a37acb 100644 --- a/artifactory/commands/transferfiles/transferfileprogress.go +++ b/artifactory/commands/transferfiles/transferfileprogress.go @@ -31,10 +31,10 @@ type TransferProgressMng struct { speedBar *progressbar.TasksProgressBar // A bar showing the estimated remaining time for the transfer timeEstBar *progressbar.TasksProgressBar + // A bar showing the number of delayed artifacts in the process + delayedBar *progressbar.TasksProgressBar // A bar showing the number of transfer failures in the process errorBar *progressbar.TasksProgressBar - // shows a note to the user if errors exists - errorNote *progressbar.TasksProgressBar // Current repo progress bars currentRepoHeadline *mpb.Bar emptyLine *mpb.Bar @@ -71,9 +71,9 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar() transfer.speedBar = transfer.transferMng.NewSpeedProgBar() transfer.timeEstBar = transfer.transferMng.NewTimeEstBar() + transfer.delayedBar = transfer.transferMng.NewDelayedBar() // Init global error count for the process transfer.errorBar = transfer.transferMng.NewErrorBar() - transfer.errorNote = transfer.transferMng.NewErrorNote() tdc.progressbar = &transfer return nil } @@ -220,6 +220,13 @@ func (t *TransferProgressMng) RemoveRepository() { time.Sleep(progressbar.ProgressRefreshRate) } +func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) { + if t.ShouldDisplay() { + diff := int64(n) + t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff) + } +} + func (t *TransferProgressMng) changeNumberOfFailuresBy(n int) { if t.ShouldDisplay() { diff := int64(n) @@ -244,7 +251,7 @@ func (t *TransferProgressMng) StopGracefully() { } func (t *TransferProgressMng) abortMetricsBars() { - for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.errorBar, t.errorNote, t.speedBar, t.timeEstBar, t.totalSize} { + for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} { if barPtr != nil { barPtr.GetBar().Abort(true) } diff --git a/utils/progressbar/progressbarmng.go b/utils/progressbar/progressbarmng.go index f3feec7a6..bb7cb26f6 100644 --- a/utils/progressbar/progressbarmng.go +++ b/utils/progressbar/progressbarmng.go @@ -250,14 +250,14 @@ func (bm *ProgressBarMng) newTasksProgressBar(getVal func() (numerator, denomina } // Initializing a counter progress bar -func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err error), headLine string) *TasksProgressBar { +func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err error), headLine string, counterDescription decor.Decorator) *TasksProgressBar { pb := &TasksProgressBar{} pb.bar = bm.container.Add(0, nil, mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name(headLine), - decor.Any(func(statistics decor.Statistics) string { + decor.Any(func(decor.Statistics) string { value, err := getVal() if err != nil { log.Error(err) @@ -266,6 +266,7 @@ func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err er return color.Green.Render(s1) }), ), + mpb.AppendDecorators(counterDescription), ) return pb } diff --git a/utils/progressbar/transferprogressbarmanager.go b/utils/progressbar/transferprogressbarmanager.go index f94a812cf..676b07b6d 100644 --- a/utils/progressbar/transferprogressbarmanager.go +++ b/utils/progressbar/transferprogressbarmanager.go @@ -1,34 +1,38 @@ package progressbar import ( + "sync" + "time" + "github.com/gookit/color" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-client-go/utils/log" - "sync" - "time" + "github.com/vbauerster/mpb/v7/decor" ) const ( - phase1HeadLine = "Phase 1: Transferring all files in the repository" - phase2HeadLine = "Phase 2: Transferring newly created and modified files" + phase1HeadLine = "Phase 1: Transferring all files in the repository" + phase2HeadLine = "Phase 2: Transferring newly created and modified files" + DelayedFilesContentNote = "Files to be transferred last, after all other files" + RetryFailureContentNote = "In Phase 3 and in subsequent executions, we'll retry transferring the failed files" ) type transferLabels struct { - Repositories string - Files string - Storage string - Note string - RetryFailureContentNote string - TransferSpeed string - EstimatedTime string - TransferFailures string - WorkingThreads string - RunningFor string - DiffStorage string - DiffFiles string - FailedStorage string - FailedFiles string + Repositories string + Files string + Storage string + Note string + TransferSpeed string + EstimatedTime string + DelayedFiles string + TransferFailures string + WorkingThreads string + RunningFor string + DiffStorage string + DiffFiles string + FailedStorage string + FailedFiles string } func formatString(emoji, key string, windows bool) string { @@ -43,13 +47,13 @@ func formatString(emoji, key string, windows bool) string { func initSProgressBarLabels(windows bool) transferLabels { pbs := transferLabels{} - pbs.RetryFailureContentNote = "In Phase 3 and in subsequent executions, we'll retry transferring the failed files." pbs.Repositories = formatString("๐Ÿ“ฆ", " Repositories", windows) pbs.Files = formatString("๐Ÿ“„", " Files", windows) pbs.Storage = formatString("๐Ÿ—„ ", " Storage", windows) pbs.Note = formatString(" ๐ŸŸ ", " Note: ", windows) pbs.TransferSpeed = formatString(" โšก", " Transfer speed: ", windows) pbs.EstimatedTime = formatString(" โŒ›", " Estimated time remaining: ", windows) + pbs.DelayedFiles = formatString(" โœ‹", " Delayed files: ", windows) pbs.TransferFailures = formatString(" โŒ", " Transfer failures: ", windows) pbs.WorkingThreads = formatString(" ๐Ÿงต", " Working threads: ", windows) pbs.RunningFor = formatString(" ๐Ÿƒ๐Ÿผ", " Running for: ", windows) @@ -91,15 +95,15 @@ func (tpm *TransferProgressMng) StopGlobalProgressBars() { } func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg { - getVals := func() (transferredStorage, totalStorage, transferresFiles, totalFiles *int64, err error) { - err = tpm.stateMng.Action(func(state *state.TransferState) error { + getVals := func() (transferredStorage, totalStorage, transferredFiles, totalFiles *int64, err error) { + err = tpm.stateMng.Action(func(*state.TransferState) error { transferredStorage = &tpm.stateMng.CurrentRepo.Phase1Info.TransferredSizeBytes totalStorage = &tpm.stateMng.CurrentRepo.Phase1Info.TotalSizeBytes - transferresFiles = &tpm.stateMng.CurrentRepo.Phase1Info.TransferredUnits + transferredFiles = &tpm.stateMng.CurrentRepo.Phase1Info.TransferredUnits totalFiles = &tpm.stateMng.CurrentRepo.Phase1Info.TotalUnits return nil }) - return transferredStorage, totalStorage, transferresFiles, totalFiles, err + return transferredStorage, totalStorage, transferredFiles, totalFiles, err } pb := tpm.barMng.newDoubleHeadLineProgressBar(phase1HeadLine, tpm.transferLabels.Storage, tpm.transferLabels.Files, getVals) @@ -130,7 +134,7 @@ func (tpm *TransferProgressMng) NewPhase1ProgressBar() *TasksWithHeadlineProg { func (tpm *TransferProgressMng) NewPhase2ProgressBar() *TasksWithHeadlineProg { getVals := func() (transferresStorage, totalStorage, transferredFiles, totalFiles *int64, err error) { - err = tpm.stateMng.Action(func(state *state.TransferState) error { + err = tpm.stateMng.Action(func(*state.TransferState) error { transferresStorage = &tpm.stateMng.CurrentRepo.Phase2Info.TransferredSizeBytes totalStorage = &tpm.stateMng.CurrentRepo.Phase2Info.TotalSizeBytes transferredFiles = &tpm.stateMng.CurrentRepo.Phase2Info.TransferredUnits @@ -168,7 +172,7 @@ func (tpm *TransferProgressMng) NewPhase2ProgressBar() *TasksWithHeadlineProg { func (tpm *TransferProgressMng) NewPhase3ProgressBar() *TasksWithHeadlineProg { getVals := func() (transferredStorage, totalStorage, transferredFiles, totalFiles *int64, err error) { - err = tpm.stateMng.Action(func(state *state.TransferState) error { + err = tpm.stateMng.Action(func(*state.TransferState) error { transferredStorage = &tpm.stateMng.CurrentRepo.Phase3Info.TransferredSizeBytes totalStorage = &tpm.stateMng.CurrentRepo.Phase3Info.TotalSizeBytes transferredFiles = &tpm.stateMng.CurrentRepo.Phase3Info.TransferredUnits @@ -236,8 +240,8 @@ func (tpm *TransferProgressMng) NewRepositoriesProgressBar() *TasksWithHeadlineP func (tpm *TransferProgressMng) NewGeneralProgBar() *TasksProgressBar { getVals := func() (transferredStorage, totalStorage, transferredFiles, totalFiles *int64, err error) { - err = tpm.stateMng.Action(func(state *state.TransferState) error { - transferredStorage = &tpm.stateMng.TransferRunStatus.OverallTransfer.TransferredSizeBytes + err = tpm.stateMng.Action(func(*state.TransferState) error { + transferredStorage = &tpm.stateMng.OverallTransfer.TransferredSizeBytes totalStorage = &tpm.stateMng.OverallTransfer.TotalSizeBytes transferredFiles = &tpm.stateMng.OverallTransfer.TransferredUnits totalFiles = &tpm.stateMng.OverallTransfer.TotalUnits @@ -268,14 +272,14 @@ func (tpm *TransferProgressMng) NewGeneralProgBar() *TasksProgressBar { func (tpm *TransferProgressMng) NewWorkingThreadsProg() *TasksProgressBar { getVal := func() (workingThreadsNum int, err error) { - err = tpm.stateMng.Action(func(state *state.TransferState) error { + err = tpm.stateMng.Action(func(*state.TransferState) error { workingThreadsNum = tpm.stateMng.WorkingThreads return nil }) return workingThreadsNum, err } - return tpm.barMng.newCounterProgressBar(getVal, tpm.transferLabels.WorkingThreads) + return tpm.barMng.newCounterProgressBar(getVal, tpm.transferLabels.WorkingThreads, nil) } func (tpm *TransferProgressMng) GetBarMng() *ProgressBarMng { @@ -283,52 +287,58 @@ func (tpm *TransferProgressMng) GetBarMng() *ProgressBarMng { } func (tpm *TransferProgressMng) NewRunningTimeProgressBar() *TasksProgressBar { - pb := tpm.barMng.NewStringProgressBar(tpm.transferLabels.RunningFor, func() string { + return tpm.barMng.NewStringProgressBar(tpm.transferLabels.RunningFor, func() string { runningTime, isRunning, err := state.GetRunningTime() if err != nil || !isRunning { runningTime = "Running time not available" } return color.Green.Render(runningTime) }) - return pb } func (tpm *TransferProgressMng) NewSpeedProgBar() *TasksProgressBar { - pb := tpm.barMng.NewStringProgressBar(tpm.transferLabels.TransferSpeed, func() string { - return color.Green.Render(tpm.stateMng.TimeEstimationManager.GetSpeedString()) + return tpm.barMng.NewStringProgressBar(tpm.transferLabels.TransferSpeed, func() string { + return color.Green.Render(tpm.stateMng.GetSpeedString()) }) - return pb } func (tpm *TransferProgressMng) NewTimeEstBar() *TasksProgressBar { - pb := tpm.barMng.NewStringProgressBar(tpm.transferLabels.EstimatedTime, func() string { - return color.Green.Render(tpm.stateMng.TimeEstimationManager.GetEstimatedRemainingTimeString()) + return tpm.barMng.NewStringProgressBar(tpm.transferLabels.EstimatedTime, func() string { + return color.Green.Render(tpm.stateMng.GetEstimatedRemainingTimeString()) }) - return pb } -func (tpm *TransferProgressMng) NewErrorBar() *TasksProgressBar { - getVals := func() (errnums int, err error) { - errnums = 0 - if !tpm.ignoreState { - errnums = int(tpm.stateMng.TransferFailures) +func (tpm *TransferProgressMng) NewDelayedBar() *TasksProgressBar { + getVals := func() (int, error) { + if tpm.ignoreState { + return 0, nil } - return errnums, err + return int(tpm.stateMng.DelayedFiles), nil } - pb := tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.TransferFailures) - return pb + counterDescription := func() string { return DelayedFilesContentNote } + return tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.DelayedFiles, tpm.createCounterDescription(counterDescription)) } -func (tpm *TransferProgressMng) NewErrorNote() *TasksProgressBar { - getString := func() (s string) { - s = "" - if tpm.stateMng.TransferFailures > 0 { - s = tpm.transferLabels.RetryFailureContentNote +func (tpm *TransferProgressMng) NewErrorBar() *TasksProgressBar { + getVals := func() (transferFailures int, err error) { + if tpm.ignoreState { + return 0, nil } - return s + return int(tpm.stateMng.TransferFailures), nil } - pb := tpm.barMng.NewStringProgressBar("", getString) - return pb + counterDescription := func() string { + if tpm.ignoreState || tpm.stateMng.TransferFailures == 0 { + return "" + } + return RetryFailureContentNote + } + return tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.TransferFailures, tpm.createCounterDescription(counterDescription)) +} + +func (tpm *TransferProgressMng) createCounterDescription(getVal func() (value string)) decor.Decorator { + return decor.Any(func(decor.Statistics) string { + return color.Gray.Render(getVal()) + }) } func (tpm *TransferProgressMng) WaitForPhasesGoRoutinesToFinish() {