Merge remote-tracking branch 'origin/main' into lists-handler

.github/actions/backup-restore-test/action.yml

@@ -99,7 +99,7 @@ runs:
         ./sanity-test restore ${{ inputs.service }}
 
     - name: Export ${{ inputs.service }} ${{ inputs.kind }}
-      if: inputs.with-export == true
+      if: ${{ inputs.with-export == 'true' }}
       id: export
       shell: bash
       working-directory: src
@@ -119,7 +119,7 @@ runs:
           cat /tmp/corsologs
 
     - name: Check export ${{ inputs.service }} ${{ inputs.kind }}
-      if: inputs.with-export == true
+      if: ${{ inputs.with-export == 'true' }}
       shell: bash
       working-directory: src
       env:
@@ -134,7 +134,7 @@ runs:
         ./sanity-test export ${{ inputs.service }}
 
     - name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
-      if: inputs.with-export == true
+      if: ${{ inputs.with-export == 'true' }}
       id: export-archive
       shell: bash
       working-directory: src
@@ -157,7 +157,7 @@ runs:
           cat /tmp/corsologs
 
     - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }}
-      if: inputs.with-export == true
+      if: ${{ inputs.with-export == 'true' }}
       shell: bash
       working-directory: src
       env:

.github/workflows/sanity-test.yaml

@@ -196,6 +196,7 @@ jobs:
           restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup
         timeout-minutes: 30
@@ -209,6 +210,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Non delta backup
         timeout-minutes: 30
@@ -222,6 +224,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup after non-delta
         timeout-minutes: 30
@@ -235,6 +238,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
 
 ##########################################################################################################################################
@@ -391,7 +395,7 @@ jobs:
           log-dir: ${{ env.CORSO_LOG_DIR }}
           with-export: true
 
-      # generate some more enteries for incremental check
+      # generate some more entries for incremental check
       - name: Groups - Create new data (for incremental)
         timeout-minutes: 30
         working-directory: ./src/cmd/factory

CHANGELOG.md

@@ -8,16 +8,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased] (beta)
 
 ### Added
-- Added export support for emails in exchange backups as `.eml` files
-- More colorful and informational cli output
+### Changed
+### Fixed
+
+## [v0.16.0] (beta) - 2023-11-28
+
+### Added
+- Export support for emails in exchange backups as `.eml` files.
+- More colorful and informational cli display.
 
 ### Changed
-- Change file extension of messages export to json to match the content
+- The file extension in Teams messages exports has switched to json to match the content type.
 - SDK consumption of the /services/m365 package has shifted from independent functions to a client-based api.
 - SDK consumers can now configure the /services/m365 graph api client configuration when constructing a new m365 client.
+- Dynamic api rate limiting allows small-scale Exchange backups to complete more quickly.
+- Kopia's local config files now uses unique filenames that match Corso configurations.  This can protect concurrent Corso operations from mistakenly clobbering storage configs during runtime.
 
 ### Fixed
-- Handle OneDrive folders being deleted and recreated midway through a backup
+- Handle OneDrive folders being deleted and recreated midway through a backup.
 - Automatically re-run a full delta query on incremental if the prior backup is found to have malformed prior-state information.
 - Retry drive item permission downloads during long-running backups after the jwt token expires and refreshes.
 - Retry item downloads during connection timeouts.

src/cmd/factory/impl/common.go

@@ -111,7 +111,12 @@ func generateAndRestoreItems(
 		Selector:          sel,
 	}
 
-	deets, _, err := ctrl.ConsumeRestoreCollections(
+	handler, err := ctrl.NewServiceHandler(opts, service)
+	if err != nil {
+		return nil, clues.Stack(err)
+	}
+
+	deets, _, err := handler.ConsumeRestoreCollections(
 		ctx,
 		rcc,
 		dataColls,
@@ -428,13 +433,6 @@ func generateAndRestoreDriveItems(
 		return nil, err
 	}
 
-	// collections := getCollections(
-	// 	service,
-	// 	tenantID,
-	// 	[]string{resourceOwner},
-	// 	input,
-	// 	version.Backup)
-
 	opts := control.DefaultOptions()
 	restoreCfg.IncludePermissions = true
 
@@ -462,7 +460,12 @@ func generateAndRestoreDriveItems(
 		Selector:          sel,
 	}
 
-	deets, _, err := ctrl.ConsumeRestoreCollections(
+	handler, err := ctrl.NewServiceHandler(opts, service)
+	if err != nil {
+		return nil, clues.Stack(err)
+	}
+
+	deets, _, err := handler.ConsumeRestoreCollections(
 		ctx,
 		rcc,
 		collections,

src/cmd/sanity_test/common/filepath.go

@@ -58,6 +58,7 @@ func BuildFilepathSanitree(
 				Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
 			}
 		} else {
+			node.CountLeaves++
 			node.Leaves[info.Name()] = &Sanileaf[fs.FileInfo, fs.FileInfo]{
 				Parent: node,
 				Self:   info,

src/cmd/sanity_test/export/exchange.go

@@ -0,0 +1,64 @@
+package export
+
+import (
+	"context"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/alcionai/corso/src/cmd/sanity_test/common"
+	"github.com/alcionai/corso/src/cmd/sanity_test/restore"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
+)
+
+func CheckEmailExport(
+	ctx context.Context,
+	ac api.Client,
+	envs common.Envs,
+) {
+	sourceTree := restore.BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
+
+	emailsExportDir := filepath.Join(envs.RestoreContainer, "Emails")
+	exportedTree := common.BuildFilepathSanitree(ctx, emailsExportDir)
+
+	ctx = clues.Add(
+		ctx,
+		"export_container_id", exportedTree.ID,
+		"export_container_name", exportedTree.Name,
+		"source_container_id", sourceTree.ID,
+		"source_container_name", sourceTree.Name)
+
+	comparator := func(
+		ctx context.Context,
+		expect *common.Sanitree[models.MailFolderable, any],
+		result *common.Sanitree[fs.FileInfo, fs.FileInfo],
+	) {
+		modifiedExpectedLeaves := map[string]*common.Sanileaf[models.MailFolderable, any]{}
+		modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
+
+		for key, val := range expect.Leaves {
+			val.Size = 0 // we cannot match up sizes
+			modifiedExpectedLeaves[key] = val
+		}
+
+		for key, val := range result.Leaves {
+			fixedName := strings.TrimSuffix(key, ".eml")
+			val.Size = 0
+
+			modifiedResultLeaves[fixedName] = val
+		}
+
+		common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil)
+	}
+
+	common.CompareDiffTrees(
+		ctx,
+		sourceTree,
+		exportedTree.Children[envs.SourceContainer],
+		comparator)
+
+	common.Infof(ctx, "Success")
+}

src/cmd/sanity_test/export/groups.go

@@ -3,6 +3,8 @@ package export
 import (
 	"context"
 	"io/fs"
+	"path/filepath"
+	"strings"
 
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
@@ -21,20 +23,30 @@ func CheckGroupsExport(
 	// assumes we only need to sanity check the default site.
 	// should we expand this to check all sites in the group?
 	// are we backing up / restoring more than the default site?
+	site, err := ac.Sites().GetByID(ctx, envs.TeamSiteID, api.CallConfig{})
+	if err != nil {
+		common.Fatal(ctx, "getting the drive:", err)
+	}
+
 	drive, err := ac.Sites().GetDefaultDrive(ctx, envs.TeamSiteID)
 	if err != nil {
 		common.Fatal(ctx, "getting the drive:", err)
 	}
 
-	driveish.CheckExport(
+	checkChannelMessagesExport(
 		ctx,
 		ac,
-		drive,
 		envs)
 
-	checkChannelMessagesExport(
+	envs.RestoreContainer = filepath.Join(
+		envs.RestoreContainer,
+		"Libraries",
+		ptr.Val(site.GetName()),
+		"Documents") // check in default loc
+	driveish.CheckExport(
 		ctx,
 		ac,
+		drive,
 		envs)
 }
 
@@ -55,7 +67,19 @@ func checkChannelMessagesExport(
 		expect *common.Sanitree[models.Channelable, models.ChatMessageable],
 		result *common.Sanitree[fs.FileInfo, fs.FileInfo],
 	) {
-		common.CompareLeaves(ctx, expect.Leaves, result.Leaves, nil)
+		for key := range expect.Leaves {
+			expect.Leaves[key].Size = 0 // msg sizes cannot be compared
+		}
+
+		updatedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
+
+		for key, leaf := range result.Leaves {
+			key = strings.TrimSuffix(key, ".json")
+			leaf.Size = 0 // we cannot compare sizes
+			updatedResultLeaves[key] = leaf
+		}
+
+		common.CompareLeaves(ctx, expect.Leaves, updatedResultLeaves, nil)
 	}
 
 	common.CompareDiffTrees(
@@ -107,7 +131,21 @@ func populateMessagesSanitree(
 			common.Fatal(ctx, "getting channel messages", err)
 		}
 
+		filteredMsgs := []models.ChatMessageable{}
+
 		for _, msg := range msgs {
+			// filter out system messages (we don't really work with them)
+			if api.IsNotSystemMessage(msg) {
+				filteredMsgs = append(filteredMsgs, msg)
+			}
+		}
+
+		if len(filteredMsgs) == 0 {
+			common.Infof(ctx, "skipped empty channel: %s", ptr.Val(ch.GetDisplayName()))
+			continue
+		}
+
+		for _, msg := range filteredMsgs {
 			child.Leaves[ptr.Val(msg.GetId())] = &common.Sanileaf[
 				models.Channelable,
 				models.ChatMessageable,

src/cmd/sanity_test/export/sharepoint.go

@@ -2,6 +2,7 @@ package export
 
 import (
 	"context"
+	"path/filepath"
 
 	"github.com/alcionai/corso/src/cmd/sanity_test/common"
 	"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
@@ -18,6 +19,7 @@ func CheckSharePointExport(
 		common.Fatal(ctx, "getting the drive:", err)
 	}
 
+	envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc
 	driveish.CheckExport(
 		ctx,
 		ac,

src/cmd/sanity_test/restore/exchange.go

@@ -19,8 +19,8 @@ func CheckEmailRestoration(
 	ac api.Client,
 	envs common.Envs,
 ) {
-	restoredTree := buildSanitree(ctx, ac, envs.UserID, envs.RestoreContainer)
-	sourceTree := buildSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
+	restoredTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.RestoreContainer)
+	sourceTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
 
 	ctx = clues.Add(
 		ctx,
@@ -39,7 +39,7 @@ func CheckEmailRestoration(
 	common.Infof(ctx, "Success")
 }
 
-func buildSanitree(
+func BuildEmailSanitree(
 	ctx context.Context,
 	ac api.Client,
 	userID, folderName string,
@@ -69,9 +69,37 @@ func buildSanitree(
 		ID:          ptr.Val(mmf.GetId()),
 		Name:        ptr.Val(mmf.GetDisplayName()),
 		CountLeaves: int(ptr.Val(mmf.GetTotalItemCount())),
+		Leaves:      map[string]*common.Sanileaf[models.MailFolderable, any]{},
 		Children:    map[string]*common.Sanitree[models.MailFolderable, any]{},
 	}
 
+	mails, err := ac.Mail().GetItemsInContainer(
+		ctx,
+		userID,
+		root.ID)
+	if err != nil {
+		common.Fatal(ctx, "getting child containers", err)
+	}
+
+	if len(mails) != root.CountLeaves {
+		common.Fatal(
+			ctx,
+			"mails count mismatch",
+			clues.New("mail message count mismatch from API"))
+	}
+
+	for _, mail := range mails {
+		m := &common.Sanileaf[models.MailFolderable, any]{
+			Parent: root,
+			Self:   mail,
+			ID:     ptr.Val(mail.GetId()),
+			Name:   ptr.Val(mail.GetSubject()),
+			Size:   int64(len(ptr.Val(mail.GetBody().GetContent()))),
+		}
+
+		root.Leaves[m.ID] = m
+	}
+
 	recursivelyBuildTree(ctx, ac, root, userID, root.Name+"/")
 
 	return root
@@ -94,6 +122,11 @@ func recursivelyBuildTree(
 	}
 
 	for _, child := range childFolders {
+		if int(ptr.Val(child.GetTotalItemCount())) == 0 {
+			common.Infof(ctx, "skipped empty folder: %s/%s", location, ptr.Val(child.GetDisplayName()))
+			continue
+		}
+
 		c := &common.Sanitree[models.MailFolderable, any]{
 			Parent:      stree,
 			Self:        child,