Enhance job summary with details about file/folders (#1952)

Azure · Dec 1, 2022 · ffbeb92 · ffbeb92
1 parent 1e8fc79
commit ffbeb92
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 17 deletions.
diff --git a/cmd/copy.go b/cmd/copy.go
@@ -1701,9 +1701,12 @@ Elapsed Time (Minutes): %v
 Number of File Transfers: %v
 Number of Folder Property Transfers: %v
 Total Number of Transfers: %v
-Number of Transfers Completed: %v
-Number of Transfers Failed: %v
-Number of Transfers Skipped: %v
+Number of File Transfers Completed: %v
+Number of Folder Transfers Completed: %v
+Number of File Transfers Failed: %v
+Number of Folder Transfers Failed: %v
+Number of File Transfers Skipped: %v
+Number of Folder Transfers Skipped: %v
 TotalBytesTransferred: %v
 Final Job Status: %v%s%s
 `,
@@ -1712,9 +1715,12 @@ Final Job Status: %v%s%s
 					summary.FileTransfers,
 					summary.FolderPropertyTransfers,
 					summary.TotalTransfers,
-					summary.TransfersCompleted,
-					summary.TransfersFailed,
-					summary.TransfersSkipped,
+					summary.TransfersCompleted-summary.FoldersCompleted,
+					summary.FoldersCompleted,
+					summary.TransfersFailed-summary.FoldersFailed,
+					summary.FoldersFailed,
+					summary.TransfersSkipped-summary.FoldersSkipped,
+					summary.FoldersSkipped,
 					summary.TotalBytesTransferred,
 					summary.JobStatus,
 					screenStats,

diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go
@@ -152,15 +152,33 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) (t
 				return string(jsonOutput)
 			} else {
 				return fmt.Sprintf(
-					"\n\nJob %s summary\nElapsed Time (Minutes): %v\nNumber of File Transfers: %v\nNumber of Folder Property Transfers: %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n",
+					`
+
+Job %s summary
+Elapsed Time (Minutes): %v
+Number of File Transfers: %v
+Number of Folder Property Transfers: %v
+Total Number Of Transfers: %v
+Number of File Transfers Completed: %v
+Number of Folder Transfers Completed: %v
+Number of File Transfers Failed: %v
+Number of Folder Transfers Failed: %v
+Number of File Transfers Skipped: %v
+Number of Folder Transfers Skipped: %v
+TotalBytesTransferred: %v
+Final Job Status: %v
+`,
 					summary.JobID.String(),
 					jobsAdmin.ToFixed(duration.Minutes(), 4),
 					summary.FileTransfers,
 					summary.FolderPropertyTransfers,
 					summary.TotalTransfers,
-					summary.TransfersCompleted,
-					summary.TransfersFailed,
-					summary.TransfersSkipped,
+					summary.TransfersCompleted-summary.FoldersCompleted,
+					summary.FoldersCompleted,
+					summary.TransfersFailed-summary.FoldersFailed,
+					summary.FoldersFailed,
+					summary.TransfersSkipped-summary.FoldersSkipped,
+					summary.FoldersSkipped,
 					summary.TotalBytesTransferred,
 					summary.JobStatus)
 			}

diff --git a/cmd/jobsShow.go b/cmd/jobsShow.go
@@ -147,14 +147,30 @@ func PrintJobProgressSummary(summary common.ListJobSummaryResponse) {
 		}
 
 		return fmt.Sprintf(
-			"\nJob %s summary\nNumber of File Transfers: %v\nNumber of Folder Property Transfers: %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nPercent Complete (approx): %.1f\nFinal Job Status: %v\n",
+			`
+Job %s summary
+Number of File Transfers: %v
+Number of Folder Property Transfers: %v
+Total Number Of Transfers: %v
+Number of File Transfers Completed: %v
+Number of Folder Transfers Completed: %v
+Number of File Transfers Failed: %v
+Number of Folder Transfers Failed: %v
+Number of File Transfers Skipped: %v
+Number of Folder Transfers Skipped: %v
+Percent Complete (approx): %.1f
+Final Job Status: %v
+`,
 			summary.JobID.String(),
 			summary.FileTransfers,
 			summary.FolderPropertyTransfers,
 			summary.TotalTransfers,
-			summary.TransfersCompleted,
-			summary.TransfersFailed,
-			summary.TransfersSkipped,
+			summary.TransfersCompleted-summary.FoldersCompleted,
+			summary.FoldersCompleted,
+			summary.TransfersFailed-summary.FoldersFailed,
+			summary.FoldersFailed,
+			summary.TransfersSkipped-summary.FoldersSkipped,
+			summary.FoldersSkipped,
 			summary.PercentComplete, // noted as approx in the format string because won't include in-flight files if this Show command is run from a different process
 			summary.JobStatus,
 		)

diff --git a/common/rpc-models.go b/common/rpc-models.go
@@ -260,8 +260,11 @@ type ListJobSummaryResponse struct {
 	FileTransfers           uint32 `json:",string"`
 	FolderPropertyTransfers uint32 `json:",string"`
 
+	FoldersCompleted   uint32 `json:",string"` // Files can be figured out by TransfersCompleted - FoldersCompleted
 	TransfersCompleted uint32 `json:",string"`
+	FoldersFailed      uint32 `json:",string"`
 	TransfersFailed    uint32 `json:",string"`
+	FoldersSkipped     uint32 `json:",string"`
 	TransfersSkipped   uint32 `json:",string"`
 
 	// includes bytes sent in retries (i.e. has double counting, if there are retries) and in failed transfers

diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go
@@ -62,7 +62,7 @@ func (jm *jobMgr) statusMgrClosed() bool {
 func (jm *jobMgr) SendJobPartCreatedMsg(msg JobPartCreatedMsg) {
 	jm.jstm.partCreated <- msg
 	if msg.IsFinalPart {
-		//Inform statusManager that this is all parts we've
+		// Inform statusManager that this is all parts we've
 		close(jm.jstm.partCreated)
 	}
 }
@@ -114,10 +114,10 @@ func (jm *jobMgr) handleStatusUpdateMessage() {
 			js.TotalBytesExpected += msg.TotalBytesEnumerated
 
 		case msg, ok := <-jstm.xferDone:
-			if !ok { //Channel is closed, all transfers have been attended.
+			if !ok { // Channel is closed, all transfers have been attended.
 				jstm.xferDone = nil
 
-				//close drainXferDone so that other components can know no further updates happen
+				// close drainXferDone so that other components can know no further updates happen
 				allXferDoneHandled = true
 				close(jstm.xferDoneDrained)
 				continue
@@ -128,15 +128,24 @@ func (jm *jobMgr) handleStatusUpdateMessage() {
 
 			switch msg.TransferStatus {
 			case common.ETransferStatus.Success():
+				if msg.IsFolderProperties {
+					js.FoldersCompleted++
+				}
 				js.TransfersCompleted++
 				js.TotalBytesTransferred += msg.TransferSize
 			case common.ETransferStatus.Failed(),
 				common.ETransferStatus.TierAvailabilityCheckFailure(),
 				common.ETransferStatus.BlobTierFailure():
+				if msg.IsFolderProperties {
+					js.FoldersFailed++
+				}
 				js.TransfersFailed++
 				js.FailedTransfers = append(js.FailedTransfers, msg)
 			case common.ETransferStatus.SkippedEntityAlreadyExists(),
 				common.ETransferStatus.SkippedBlobHasSnapshots():
+				if msg.IsFolderProperties {
+					js.FoldersSkipped++
+				}
 				js.TransfersSkipped++
 				js.SkippedTransfers = append(js.SkippedTransfers, msg)
 			}