Transfer - Show delayed files (#980)

.github/workflows/analysis.yml

@@ -36,7 +36,8 @@ jobs:
         with:
           go-version: 1.20.x
 
+      # Temporarily set version 2.18.0 to workaround https://github.com/securego/gosec/issues/1046
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@master
+        uses: securego/gosec@v2.18.0
         with:
           args: -exclude G204,G301,G302,G304,G306 -tests -exclude-dir \.*test\.* ./...

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -174,6 +174,13 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 			return err
 		}
 
+		if base.progressBar != nil {
+			base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts))
+		}
+		if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
+			log.Warn("Couldn't decrease the delayed files counter", err.Error())
+		}
+
 		// Remove the file, so it won't be consumed again.
 		if err = os.Remove(filePath); err != nil {
 			return errorutils.CheckError(err)
@@ -203,6 +210,23 @@ func getDelayFiles(repoKeys []string) (filesPaths []string, err error) {
 	return getErrorOrDelayFiles(repoKeys, getJfrogTransferRepoDelaysDir)
 }
 
+func getDelayedFilesCount(repoKeys []string) (int, error) {
+	files, err := getDelayFiles(repoKeys)
+	if err != nil {
+		return -1, err
+	}
+
+	count := 0
+	for _, file := range files {
+		delayedFiles, err := readDelayFile(file)
+		if err != nil {
+			return -1, err
+		}
+		count += len(delayedFiles.DelayedArtifacts)
+	}
+	return count, nil
+}
+
 const (
 	maven  = "Maven"
 	gradle = "Gradle"
@@ -256,6 +280,12 @@ func (delayHelper delayUploadHelper) delayUploadIfNecessary(phase phaseBase, fil
 		if shouldDelay(file.Name) {
 			delayed = true
 			delayHelper.delayedArtifactsChannelMng.add(file)
+			if phase.progressBar != nil {
+				phase.progressBar.changeNumberOfDelayedFiles(1)
+			}
+			if err := phase.stateManager.ChangeDelayedFilesCountBy(1, true); err != nil {
+				log.Warn("Couldn't increase the delayed files counter", err.Error())
+			}
 		}
 	}
 	return

artifactory/commands/transferfiles/state/runstatus.go

@@ -36,6 +36,7 @@ type TransferRunStatus struct {
 	BuildInfoRepo         bool `json:"build_info_repo,omitempty"`
 	CurrentRepoPhase      int  `json:"current_repo_phase,omitempty"`
 	WorkingThreads        int  `json:"working_threads,omitempty"`
+	DelayedFiles          uint `json:"delayed_files,omitempty"`
 	TransferFailures      uint `json:"transfer_failures,omitempty"`
 	TimeEstimationManager `json:"time_estimation,omitempty"`
 	StaleChunks           []StaleChunks `json:"stale_chunks,omitempty"`

artifactory/commands/transferfiles/state/statemanager.go

@@ -263,6 +263,17 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er
 	})
 }
 
+func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error {
+	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
+		if increase {
+			transferRunStatus.DelayedFiles += count
+		} else {
+			transferRunStatus.DelayedFiles -= count
+		}
+		return nil
+	})
+}
+
 func (ts *TransferStateManager) ChangeTransferFailureCountBy(count uint, increase bool) error {
 	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
 		if increase {

artifactory/commands/transferfiles/status.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/state"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
@@ -78,9 +79,14 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings.
 	addString(output, "🧵", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2)
 	addString(output, "⚡", "Transfer speed", stateManager.GetSpeedString(), 2)
 	addString(output, "⌛", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
+	delayedTxt := strconv.FormatUint(uint64(stateManager.DelayedFiles), 10)
+	if stateManager.DelayedFiles > 0 {
+		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
+	}
+	addString(output, "✋", "Delayed files", delayedTxt, 2)
 	failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10)
 	if stateManager.TransferFailures > 0 {
-		failureTxt += " (" + "In Phase 3 and in subsequent executions, we'll retry transferring the failed files." + ")"
+		failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
 	}
 	addString(output, "❌", "Transfer failures", failureTxt, 2)
 }

artifactory/commands/transferfiles/status_test.go

@@ -69,7 +69,8 @@ func TestShowStatus(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Less than a minute")
-	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files.)")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
@@ -99,7 +100,8 @@ func TestShowStatusDiffPhase(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Not available in this phase")
-	assert.Contains(t, results, "Transfer failures:		223")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
@@ -129,6 +131,7 @@ func TestShowBuildInfoRepo(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		Not available while transferring a build-info repository")
 	assert.Contains(t, results, "Estimated time remaining:	Less than a minute")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
 	assert.Contains(t, results, "Transfer failures:		223")
 
 	// Check repository status
@@ -174,11 +177,12 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks
 	stateManager.TotalRepositories.TotalUnits = 1111
 	stateManager.TotalRepositories.TransferredUnits = 15
 	stateManager.WorkingThreads = 16
+	stateManager.DelayedFiles = 20
 	stateManager.TransferFailures = 223
 
-	stateManager.TimeEstimationManager.LastSpeeds = []float64{12}
-	stateManager.TimeEstimationManager.LastSpeedsSum = 12
-	stateManager.TimeEstimationManager.SpeedsAverage = 12
+	stateManager.LastSpeeds = []float64{12}
+	stateManager.LastSpeedsSum = 12
+	stateManager.SpeedsAverage = 12
 
 	if staleChunks {
 		stateManager.StaleChunks = append(stateManager.StaleChunks, state.StaleChunks{

artifactory/commands/transferfiles/transfer.go

@@ -228,8 +228,15 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui
 			return e
 		}
 		tdc.stateManager.TransferFailures = uint(numberInitialErrors)
+
+		numberInitialDelays, e := getDelayedFilesCount(allSourceLocalRepos)
+		if e != nil {
+			return e
+		}
+		tdc.stateManager.DelayedFiles = uint(numberInitialDelays)
 	} else {
 		tdc.stateManager.TransferFailures = 0
+		tdc.stateManager.DelayedFiles = 0
 	}
 	return nil
 }

artifactory/commands/transferfiles/transferfileprogress.go

@@ -31,10 +31,10 @@ type TransferProgressMng struct {
 	speedBar *progressbar.TasksProgressBar
 	// A bar showing the estimated remaining time for the transfer
 	timeEstBar *progressbar.TasksProgressBar
+	// A bar showing the number of delayed artifacts in the process
+	delayedBar *progressbar.TasksProgressBar
 	// A bar showing the number of transfer failures in the process
 	errorBar *progressbar.TasksProgressBar
-	// shows a note to the user if errors exists
-	errorNote *progressbar.TasksProgressBar
 	// Current repo progress bars
 	currentRepoHeadline *mpb.Bar
 	emptyLine           *mpb.Bar
@@ -71,9 +71,9 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom
 	transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar()
 	transfer.speedBar = transfer.transferMng.NewSpeedProgBar()
 	transfer.timeEstBar = transfer.transferMng.NewTimeEstBar()
+	transfer.delayedBar = transfer.transferMng.NewDelayedBar()
 	// Init global error count for the process
 	transfer.errorBar = transfer.transferMng.NewErrorBar()
-	transfer.errorNote = transfer.transferMng.NewErrorNote()
 	tdc.progressbar = &transfer
 	return nil
 }
@@ -220,6 +220,13 @@ func (t *TransferProgressMng) RemoveRepository() {
 	time.Sleep(progressbar.ProgressRefreshRate)
 }
 
+func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) {
+	if t.ShouldDisplay() {
+		diff := int64(n)
+		t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff)
+	}
+}
+
 func (t *TransferProgressMng) changeNumberOfFailuresBy(n int) {
 	if t.ShouldDisplay() {
 		diff := int64(n)
@@ -244,7 +251,7 @@ func (t *TransferProgressMng) StopGracefully() {
 }
 
 func (t *TransferProgressMng) abortMetricsBars() {
-	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.errorBar, t.errorNote, t.speedBar, t.timeEstBar, t.totalSize} {
+	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} {
 		if barPtr != nil {
 			barPtr.GetBar().Abort(true)
 		}

utils/progressbar/progressbarmng.go

@@ -250,14 +250,14 @@ func (bm *ProgressBarMng) newTasksProgressBar(getVal func() (numerator, denomina
 }
 
 // Initializing a counter progress bar
-func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err error), headLine string) *TasksProgressBar {
+func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err error), headLine string, counterDescription decor.Decorator) *TasksProgressBar {
 	pb := &TasksProgressBar{}
 	pb.bar = bm.container.Add(0,
 		nil,
 		mpb.BarRemoveOnComplete(),
 		mpb.PrependDecorators(
 			decor.Name(headLine),
-			decor.Any(func(statistics decor.Statistics) string {
+			decor.Any(func(decor.Statistics) string {
 				value, err := getVal()
 				if err != nil {
 					log.Error(err)
@@ -266,6 +266,7 @@ func (bm *ProgressBarMng) newCounterProgressBar(getVal func() (value int, err er
 				return color.Green.Render(s1)
 			}),
 		),
+		mpb.AppendDecorators(counterDescription),
 	)
 	return pb
 }