diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index 13e08803da..afcf008807 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -8,7 +8,6 @@ import ( "github.com/alcionai/clues" kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/sites" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -24,7 +23,6 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -336,70 +334,3 @@ func (suite *SharePointCollectionSuite) TestCollection_streamItems() { }) } } - -// TestRestoreListCollection verifies Graph Restore API for the List Collection -func (suite *SharePointCollectionSuite) TestListCollection_Restore() { - t := suite.T() - // https://github.com/microsoftgraph/msgraph-sdk-go/issues/490 - t.Skip("disabled until upstream issue with list restore is fixed.") - - ctx, flush := tester.NewContext(t) - defer flush() - - service := createTestService(t, suite.creds) - listing := spMock.ListDefault("Mock List") - testName := "MockListing" - listing.SetDisplayName(&testName) - byteArray, err := service.Serialize(listing) - require.NoError(t, err, clues.ToCore(err)) - - info := &details.SharePointInfo{ - ItemName: testName, - } - - listData, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(byteArray)), - testName, - details.ItemInfo{SharePoint: info}) - require.NoError(t, err, clues.ToCore(err)) - - destName := testdata.DefaultRestoreConfig("").Location - - lrh := NewListsRestoreHandler(suite.siteID, suite.ac.Lists()) - deets, err := restoreListItem(ctx, lrh, listData, suite.siteID, destName) - assert.NoError(t, err, clues.ToCore(err)) - t.Logf("List created: %s\n", deets.SharePoint.ItemName) - - // Clean-Up - var ( - builder = service.Client().Sites().BySiteId(suite.siteID).Lists() - isFound bool - deleteID string - ) - - for { - resp, err := builder.Get(ctx, nil) - assert.NoError(t, err, "getting site lists", clues.ToCore(err)) - - for _, temp := range resp.GetValue() { - if ptr.Val(temp.GetDisplayName()) == deets.SharePoint.ItemName { - isFound = true - deleteID = ptr.Val(temp.GetId()) - - break - } - } - // Get Next Link - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsRequestBuilder(link, service.Adapter()) - } - - if isFound { - err := DeleteList(ctx, service, suite.siteID, deleteID) - assert.NoError(t, err, clues.ToCore(err)) - } -} diff --git a/src/internal/m365/collection/site/lists.go b/src/internal/m365/collection/site/lists.go deleted file mode 100644 index 425698cbea..0000000000 --- a/src/internal/m365/collection/site/lists.go +++ /dev/null @@ -1,23 +0,0 @@ -package site - -import ( - "context" - - "github.com/alcionai/corso/src/pkg/services/m365/api/graph" -) - -// DeleteList removes a list object from a site. -// deletes require unique http clients -// https://github.com/alcionai/corso/issues/2707 -func DeleteList( - ctx context.Context, - gs graph.Servicer, - siteID, listID string, -) error { - err := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Delete(ctx, nil) - if err != nil { - return graph.Wrap(ctx, err, "deleting list") - } - - return nil -} diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index febc644958..c532005ceb 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -207,6 +207,11 @@ func RestoreListCollection( itemData, siteID, restoreContainerName) + if err != nil && + errors.Is(err, api.ErrCannotCreateWebTemplateExtension) { + continue + } + if err != nil { el.AddRecoverable(ctx, err) continue diff --git a/src/internal/m365/collection/site/restore_test.go b/src/internal/m365/collection/site/restore_test.go new file mode 100644 index 0000000000..0ef7e171ba --- /dev/null +++ b/src/internal/m365/collection/site/restore_test.go @@ -0,0 +1,184 @@ +package site + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/readers" + "github.com/alcionai/corso/src/internal/data" + dataMock "github.com/alcionai/corso/src/internal/data/mock" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/graph" +) + +type SharePointRestoreSuite struct { + tester.Suite + siteID string + creds account.M365Config + ac api.Client +} + +func (suite *SharePointRestoreSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + graph.InitializeConcurrencyLimiter(ctx, false, 4) + + suite.siteID = tconfig.M365SiteID(t) + a := tconfig.NewM365Account(t) + m365, err := a.M365Config() + require.NoError(t, err, clues.ToCore(err)) + + suite.creds = m365 + + ac, err := api.NewClient( + m365, + control.DefaultOptions(), + count.New()) + require.NoError(t, err, clues.ToCore(err)) + + suite.ac = ac +} + +func TestSharePointRestoreSuite(t *testing.T) { + suite.Run(t, &SharePointRestoreSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +// TestRestoreListCollection verifies Graph Restore API for the List Collection +func (suite *SharePointRestoreSuite) TestListCollection_Restore() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + testName, lrh, destName, mockData := setupDependencies( + suite, + suite.ac, + suite.siteID, + suite.creds, + "genericList") + + deets, err := restoreListItem(ctx, lrh, mockData, suite.siteID, destName) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, fmt.Sprintf("%s_%s", destName, testName), deets.SharePoint.ItemName) + + // Clean-Up + deleteList(ctx, t, suite.siteID, lrh, deets) +} + +func (suite *SharePointRestoreSuite) TestListCollection_Restore_invalidListTemplate() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + _, lrh, destName, mockData := setupDependencies( + suite, + suite.ac, + suite.siteID, + suite.creds, + api.WebTemplateExtensionsListTemplateName) + + _, err := restoreListItem(ctx, lrh, mockData, suite.siteID, destName) + require.Error(t, err) + assert.Contains(t, err.Error(), api.ErrCannotCreateWebTemplateExtension.Error()) +} + +func deleteList( + ctx context.Context, + t *testing.T, + siteID string, + lrh listsRestoreHandler, + deets details.ItemInfo, +) { + var ( + isFound bool + deleteID string + ) + + lists, err := lrh.ac.Client. + Lists(). + GetLists(ctx, siteID, api.CallConfig{}) + assert.NoError(t, err, "getting site lists", clues.ToCore(err)) + + for _, l := range lists { + if ptr.Val(l.GetDisplayName()) == deets.SharePoint.ItemName { + isFound = true + deleteID = ptr.Val(l.GetId()) + + break + } + } + + if isFound { + err := lrh.DeleteList(ctx, deleteID) + assert.NoError(t, err, clues.ToCore(err)) + } +} + +func setupDependencies( + suite tester.Suite, + ac api.Client, + siteID string, + creds account.M365Config, + listTemplate string) ( + string, listsRestoreHandler, string, *dataMock.Item, +) { + t := suite.T() + testName := "MockListing" + + lrh := NewListsRestoreHandler(siteID, ac.Lists()) + + service := createTestService(t, creds) + + listInfo := models.NewListInfo() + listInfo.SetTemplate(ptr.To(listTemplate)) + + listing := spMock.ListDefault("Mock List") + listing.SetDisplayName(&testName) + listing.SetList(listInfo) + + byteArray, err := service.Serialize(listing) + require.NoError(t, err, clues.ToCore(err)) + + destName := testdata.DefaultRestoreConfig("").Location + + listData, err := data.NewPrefetchedItemWithInfo( + io.NopCloser(bytes.NewReader(byteArray)), + testName, + details.ItemInfo{SharePoint: api.ListToSPInfo(listing)}) + require.NoError(t, err, clues.ToCore(err)) + + r, err := readers.NewVersionedRestoreReader(listData.ToReader()) + require.NoError(t, err) + + mockData := &dataMock.Item{ + ItemID: testName, + Reader: r, + } + + return testName, lrh, destName, mockData +} diff --git a/src/internal/operations/pathtransformer/restore_path_transformer.go b/src/internal/operations/pathtransformer/restore_path_transformer.go index 5ee431b53f..5cc7944793 100644 --- a/src/internal/operations/pathtransformer/restore_path_transformer.go +++ b/src/internal/operations/pathtransformer/restore_path_transformer.go @@ -142,7 +142,8 @@ func makeRestorePathsForEntry( // * OneDrive/SharePoint (needs drive information) switch true { case ent.Exchange != nil || - (ent.Groups != nil && ent.Groups.ItemType == details.GroupsChannelMessage): + (ent.Groups != nil && ent.Groups.ItemType == details.GroupsChannelMessage) || + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.SharePointList): // TODO(ashmrtn): Eventually make Events have it's own function to handle // setting the restore destination properly. res.RestorePath, err = basicLocationPath(repoRef, locRef) diff --git a/src/internal/operations/pathtransformer/restore_path_transformer_test.go b/src/internal/operations/pathtransformer/restore_path_transformer_test.go index 5dc637e158..390354ac02 100644 --- a/src/internal/operations/pathtransformer/restore_path_transformer_test.go +++ b/src/internal/operations/pathtransformer/restore_path_transformer_test.go @@ -50,16 +50,19 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { driveID = "some-drive-id" siteID = "some-site-id" extraItemName = "some-item" + listName = "list1" SharePointRootItemPath = testdata.SharePointRootPath.MustAppend(extraItemName, true) + SharePointListItemPath = testdata.SharePointListPath.MustAppend(listName, true) GroupsRootItemPath = testdata.GroupsRootPath.MustAppend(extraItemName, true) ) table := []struct { - name string - backupVersion int - input []*details.Entry - expectErr assert.ErrorAssertionFunc - expected []expectPaths + name string + backupVersion int + input []*details.Entry + expectErr assert.ErrorAssertionFunc + expected []expectPaths + isSharepointList bool }{ { name: "Groups List Errors v9", @@ -157,13 +160,13 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { }, }, { - name: "SharePoint List Errors", + name: "SharePoint List, item in root", // No version bump for the change so we always have to check for this. backupVersion: version.All8MigrateUserPNToID, input: []*details.Entry{ { - RepoRef: SharePointRootItemPath.RR.String(), - LocationRef: SharePointRootItemPath.Loc.String(), + RepoRef: SharePointListItemPath.RR.String(), + LocationRef: SharePointListItemPath.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointList, @@ -171,7 +174,14 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { }, }, }, - expectErr: assert.Error, + expected: []expectPaths{ + { + storage: SharePointListItemPath.RR.String(), + restore: toRestore(SharePointListItemPath.RR), + }, + }, + expectErr: assert.NoError, + isSharepointList: true, }, { name: "SharePoint Page Errors", @@ -413,12 +423,23 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { for _, e := range test.expected { tmp := path.RestorePaths{} - p, err := path.FromDataLayerPath(e.storage, true) + var p path.Path + var err error + + if test.isSharepointList { + p, err = path.PrefixOrPathFromDataLayerPath(e.storage, true) + } else { + p, err = path.FromDataLayerPath(e.storage, true) + } require.NoError(t, err, "parsing expected storage path", clues.ToCore(err)) tmp.StoragePath = p - p, err = path.FromDataLayerPath(e.restore, false) + if test.isSharepointList { + p, err = path.PrefixOrPathFromDataLayerPath(e.restore, false) + } else { + p, err = path.FromDataLayerPath(e.restore, false) + } require.NoError(t, err, "parsing expected restore path", clues.ToCore(err)) if e.isRestorePrefix { diff --git a/src/pkg/backup/details/testdata/testdata.go b/src/pkg/backup/details/testdata/testdata.go index ee56d36c3b..1b47620d20 100644 --- a/src/pkg/backup/details/testdata/testdata.go +++ b/src/pkg/backup/details/testdata/testdata.go @@ -16,8 +16,16 @@ import ( // mustParsePath takes a string representing a resource path and returns a path // instance. Panics if the path cannot be parsed. Useful for simple variable // assignments. -func mustParsePath(ref string, isItem bool) path.Path { - p, err := path.FromDataLayerPath(ref, isItem) +func mustParsePath(ref string, isItem, isSharepointList bool) path.Path { + var p path.Path + var err error + + if isSharepointList { + p, err = path.PrefixOrPathFromDataLayerPath(ref, isItem) + } else { + p, err = path.FromDataLayerPath(ref, isItem) + } + if err != nil { panic(err) } @@ -47,7 +55,7 @@ func locFromRepo(rr path.Path, isItem bool) *path.Builder { if rr.Service() == path.GroupsService { loc = loc.PopFront().PopFront().PopFront() - } else if rr.Service() == path.OneDriveService || rr.Category() == path.LibrariesCategory { + } else if rr.Service() == path.OneDriveService || rr.Category() == path.LibrariesCategory || rr.Category() == path.ListsCategory { loc = loc.PopFront() } @@ -117,9 +125,12 @@ func (p repoRefAndLocRef) locationAsRepoRef() path.Path { return res } -func mustPathRep(ref string, isItem bool) repoRefAndLocRef { +func mustPathRep(ref string, isItem, isSharepointList bool) repoRefAndLocRef { + var rr path.Path + var err error + res := repoRefAndLocRef{} - tmp := mustParsePath(ref, isItem) + tmp := mustParsePath(ref, isItem, isSharepointList) // Now append stuff to the RepoRef elements so we have distinct LocationRef // and RepoRef elements to simulate using IDs in the path instead of display @@ -133,12 +144,21 @@ func mustPathRep(ref string, isItem bool) repoRefAndLocRef { rrPB = rrPB.Append(tmp.Item() + fileSuffix) } - rr, err := rrPB.ToDataLayerPath( - tmp.Tenant(), - tmp.ProtectedResource(), - tmp.Service(), - tmp.Category(), - isItem) + if isSharepointList { + rr, err = rrPB.ToDataLayerSharePointListPath( + tmp.Tenant(), + tmp.ProtectedResource(), + tmp.Category(), + isItem) + } else { + rr, err = rrPB.ToDataLayerPath( + tmp.Tenant(), + tmp.ProtectedResource(), + tmp.Service(), + tmp.Category(), + isItem) + } + if err != nil { panic(err) } @@ -166,7 +186,7 @@ var ( Time3 = time.Date(2023, 9, 21, 10, 0, 0, 0, time.UTC) Time4 = time.Date(2023, 10, 21, 10, 0, 0, 0, time.UTC) - ExchangeEmailInboxPath = mustPathRep("tenant-id/exchange/user-id/email/Inbox", false) + ExchangeEmailInboxPath = mustPathRep("tenant-id/exchange/user-id/email/Inbox", false, false) ExchangeEmailBasePath = ExchangeEmailInboxPath.MustAppend("subfolder", false) ExchangeEmailBasePath2 = ExchangeEmailInboxPath.MustAppend("othersubfolder/", false) ExchangeEmailBasePath3 = ExchangeEmailBasePath2.MustAppend("subsubfolder", false) @@ -314,7 +334,7 @@ var ( }, } - ExchangeContactsRootPath = mustPathRep("tenant-id/exchange/user-id/contacts/contacts", false) + ExchangeContactsRootPath = mustPathRep("tenant-id/exchange/user-id/contacts/contacts", false, false) ExchangeContactsBasePath = ExchangeContactsRootPath.MustAppend("contacts", false) ExchangeContactsBasePath2 = ExchangeContactsRootPath.MustAppend("morecontacts", false) ExchangeContactsItemPath1 = ExchangeContactsBasePath.MustAppend(ItemName1, true) @@ -403,8 +423,8 @@ var ( }, } - ExchangeEventsBasePath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false) - ExchangeEventsBasePath2 = mustPathRep("tenant-id/exchange/user-id/events/moreholidays", false) + ExchangeEventsBasePath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false, false) + ExchangeEventsBasePath2 = mustPathRep("tenant-id/exchange/user-id/events/moreholidays", false, false) ExchangeEventsItemPath1 = ExchangeEventsBasePath.MustAppend(ItemName1, true) ExchangeEventsItemPath2 = ExchangeEventsBasePath2.MustAppend(ItemName2, true) @@ -507,7 +527,7 @@ var ( }, } - OneDriveRootPath = mustPathRep("tenant-id/onedrive/user-id/files/drives/foo/root:", false) + OneDriveRootPath = mustPathRep("tenant-id/onedrive/user-id/files/drives/foo/root:", false, false) OneDriveFolderPath = OneDriveRootPath.MustAppend("folder", false) OneDriveBasePath1 = OneDriveFolderPath.MustAppend("a", false) OneDriveBasePath2 = OneDriveFolderPath.MustAppend("b", false) @@ -732,10 +752,11 @@ var ( }, } - GroupsRootPath = mustPathRep("tenant-id/groups/group-id/libraries/sites/site-id/drives/foo/root:", false) + GroupsRootPath = mustPathRep("tenant-id/groups/group-id/libraries/sites/site-id/drives/foo/root:", false, false) - SharePointRootPath = mustPathRep("tenant-id/sharepoint/site-id/libraries/drives/foo/root:", false) + SharePointRootPath = mustPathRep("tenant-id/sharepoint/site-id/libraries/drives/foo/root:", false, false) SharePointLibraryPath = SharePointRootPath.MustAppend("library", false) + SharePointListPath = mustPathRep("tenant-id/sharepoint/site-id/lists", false, true) SharePointBasePath1 = SharePointLibraryPath.MustAppend("a", false) SharePointBasePath2 = SharePointLibraryPath.MustAppend("b", false) diff --git a/src/pkg/path/builder.go b/src/pkg/path/builder.go index 5fe8169c14..2415912796 100644 --- a/src/pkg/path/builder.go +++ b/src/pkg/path/builder.go @@ -355,6 +355,34 @@ func (pb Builder) ToDataLayerSharePointPath( return pb.ToDataLayerPath(tenant, site, SharePointService, category, isItem) } +func (pb Builder) ToDataLayerSharePointListPath( + tenant, site string, + category CategoryType, + isItem bool, +) (Path, error) { + if err := ValidateServiceAndCategory(SharePointService, category); err != nil { + return nil, err + } + + if err := verifyInputValues(tenant, site); err != nil { + return nil, err + } + + prefixItems := []string{ + tenant, + SharePointService.String(), + site, + category.String(), + } + + return &dataLayerResourcePath{ + Builder: *pb.withPrefix(prefixItems...), + service: SharePointService, + category: category, + hasItem: isItem, + }, nil +} + // --------------------------------------------------------------------------- // Stringers and PII Concealer Compliance // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go index 0f5fa189e3..bd6b2057de 100644 --- a/src/pkg/services/m365/api/lists.go +++ b/src/pkg/services/m365/api/lists.go @@ -13,6 +13,8 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +var ErrCannotCreateWebTemplateExtension = clues.New("unable to create webTemplateExtension type lists") + const ( AttachmentsColumnName = "Attachments" EditColumnName = "Edit" @@ -42,6 +44,8 @@ const ( ReadOnlyOrHiddenFieldNamePrefix = "_" DescoratorFieldNamePrefix = "@" + + WebTemplateExtensionsListTemplateName = "webTemplateExtensionsList" ) var addressFieldNames = []string{ @@ -247,6 +251,12 @@ func (c Lists) PostList( // this ensure all columns, contentTypes are set to the newList newList := ToListable(oldList, newListName) + if newList != nil && + newList.GetList() != nil && + ptr.Val(newList.GetList().GetTemplate()) == WebTemplateExtensionsListTemplateName { + return nil, clues.StackWC(ctx, ErrCannotCreateWebTemplateExtension) + } + // Restore to List base to M365 back store restoredList, err := c.Stable. Client(). diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go index 0d35b3361a..9fcfbdf02b 100644 --- a/src/pkg/services/m365/api/lists_test.go +++ b/src/pkg/services/m365/api/lists_test.go @@ -757,6 +757,38 @@ func (suite *ListsAPIIntgSuite) TestLists_PostList() { require.NoError(t, err) } +func (suite *ListsAPIIntgSuite) TestLists_PostList_invalidTemplate() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + overrideListInfo := models.NewListInfo() + overrideListInfo.SetTemplate(ptr.To(WebTemplateExtensionsListTemplateName)) + + _, list := getFieldsDataAndList() + list.SetList(overrideListInfo) + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + _, err = acl.PostList(ctx, siteID, listName, oldListByteArray) + require.Error(t, err) + assert.Equal(t, ErrCannotCreateWebTemplateExtension.Error(), err.Error()) +} + func (suite *ListsAPIIntgSuite) TestLists_DeleteList() { t := suite.T()