diff --git a/src/internal/common/keys/keys.go b/src/internal/common/keys/keys.go new file mode 100644 index 0000000000..07d0cf4444 --- /dev/null +++ b/src/internal/common/keys/keys.go @@ -0,0 +1,31 @@ +package keys + +type Set map[string]struct{} + +func (ks Set) HasKey(key string) bool { + if _, ok := ks[key]; ok { + return true + } + + return false +} + +func (ks Set) Keys() []string { + sliceKeys := make([]string, 0) + + for k := range ks { + sliceKeys = append(sliceKeys, k) + } + + return sliceKeys +} + +func HasKeys(data map[string]any, keys ...string) bool { + for _, k := range keys { + if _, ok := data[k]; !ok { + return false + } + } + + return true +} diff --git a/src/internal/common/keys/keys_test.go b/src/internal/common/keys/keys_test.go new file mode 100644 index 0000000000..41e6de6d97 --- /dev/null +++ b/src/internal/common/keys/keys_test.go @@ -0,0 +1,122 @@ +package keys + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type KeySetTestSuite struct { + tester.Suite +} + +func TestKeySetTestSuite(t *testing.T) { + suite.Run(t, &KeySetTestSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *KeySetTestSuite) TestHasKey() { + tests := []struct { + name string + keySet Set + key string + expect assert.BoolAssertionFunc + }{ + { + name: "key exists in the set", + keySet: Set{"key1": {}, "key2": {}}, + key: "key1", + expect: assert.True, + }, + { + name: "key does not exist in the set", + keySet: Set{"key1": {}, "key2": {}}, + key: "nonexistent", + expect: assert.False, + }, + { + name: "empty set", + keySet: Set{}, + key: "key", + expect: assert.False, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + test.expect(suite.T(), test.keySet.HasKey(test.key)) + }) + } +} + +func (suite *KeySetTestSuite) TestKeys() { + tests := []struct { + name string + keySet Set + expect assert.ValueAssertionFunc + }{ + { + name: "non-empty set", + keySet: Set{"key1": {}, "key2": {}}, + expect: assert.NotEmpty, + }, + { + name: "empty set", + keySet: Set{}, + expect: assert.Empty, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + keys := test.keySet.Keys() + test.expect(suite.T(), keys, []string{"key1", "key2"}) + }) + } +} + +func (suite *KeySetTestSuite) TestHasKeys() { + tests := []struct { + name string + data map[string]any + keys []string + expect assert.BoolAssertionFunc + }{ + { + name: "has all keys", + data: map[string]any{ + "key1": "data1", + "key2": 2, + "key3": struct{}{}, + }, + keys: []string{"key1", "key2", "key3"}, + expect: assert.True, + }, + { + name: "has some keys", + data: map[string]any{ + "key1": "data1", + "key2": 2, + }, + keys: []string{"key1", "key2", "key3"}, + expect: assert.False, + }, + { + name: "has no key", + data: map[string]any{ + "key1": "data1", + "key2": 2, + }, + keys: []string{"key4", "key5", "key6"}, + expect: assert.False, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + test.expect(suite.T(), HasKeys(test.data, test.keys...)) + }) + } +} diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index a177a04246..79be31f322 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -139,7 +139,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { }, getItem: func(t *testing.T, itemName string) data.Item { byteArray := spMock.Page(itemName) - page, err := betaAPI.CreatePageFromBytes(byteArray) + page, err := betaAPI.BytesToSitePageable(byteArray) require.NoError(t, err, clues.ToCore(err)) data, err := data.NewPrefetchedItemWithInfo( @@ -307,7 +307,8 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { destName := testdata.DefaultRestoreConfig("").Location - deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName) + lrh := NewListsRestoreHandler(suite.siteID, suite.ac.Lists()) + deets, err := restoreListItem(ctx, lrh, listData, suite.siteID, destName) assert.NoError(t, err, clues.ToCore(err)) t.Logf("List created: %s\n", deets.SharePoint.ItemName) diff --git a/src/internal/m365/collection/site/handlers.go b/src/internal/m365/collection/site/handlers.go index c03d84e287..05848c3235 100644 --- a/src/internal/m365/collection/site/handlers.go +++ b/src/internal/m365/collection/site/handlers.go @@ -20,3 +20,23 @@ type getItemByIDer interface { type getItemser interface { GetItems(ctx context.Context, cc api.CallConfig) ([]models.Listable, error) } + +type restoreHandler interface { + PostLister + DeleteLister +} + +type PostLister interface { + PostList( + ctx context.Context, + listName string, + storedListData []byte, + ) (models.Listable, error) +} + +type DeleteLister interface { + DeleteList( + ctx context.Context, + listID string, + ) error +} diff --git a/src/internal/m365/collection/site/lists_handler.go b/src/internal/m365/collection/site/lists_handler.go index b10f961596..145db89601 100644 --- a/src/internal/m365/collection/site/lists_handler.go +++ b/src/internal/m365/collection/site/lists_handler.go @@ -29,3 +29,32 @@ func (bh listsBackupHandler) GetItemByID(ctx context.Context, itemID string) (mo func (bh listsBackupHandler) GetItems(ctx context.Context, cc api.CallConfig) ([]models.Listable, error) { return bh.ac.GetLists(ctx, bh.protectedResource, cc) } + +var _ restoreHandler = &listsRestoreHandler{} + +type listsRestoreHandler struct { + ac api.Lists + protectedResource string +} + +func NewListsRestoreHandler(protectedResource string, ac api.Lists) listsRestoreHandler { + return listsRestoreHandler{ + ac: ac, + protectedResource: protectedResource, + } +} + +func (rh listsRestoreHandler) PostList( + ctx context.Context, + listName string, + storedListData []byte, +) (models.Listable, error) { + return rh.ac.PostList(ctx, rh.protectedResource, listName, storedListData) +} + +func (rh listsRestoreHandler) DeleteList( + ctx context.Context, + listID string, +) error { + return rh.ac.DeleteList(ctx, rh.protectedResource, listID) +} diff --git a/src/internal/m365/collection/site/mock/list.go b/src/internal/m365/collection/site/mock/list.go index 30b4d50b15..b2727f6950 100644 --- a/src/internal/m365/collection/site/mock/list.go +++ b/src/internal/m365/collection/site/mock/list.go @@ -9,15 +9,33 @@ import ( ) type ListHandler struct { - ListItem models.Listable - Err error + List models.Listable + Err error } func (lh *ListHandler) GetItemByID(ctx context.Context, itemID string) (models.Listable, error) { ls := models.NewList() - lh.ListItem = ls - lh.ListItem.SetId(ptr.To(itemID)) + lh.List = ls + lh.List.SetId(ptr.To(itemID)) return ls, lh.Err } + +type ListRestoreHandler struct { + List models.Listable + Err error +} + +func (lh *ListRestoreHandler) PostList( + ctx context.Context, + listName string, + storedListBytes []byte, +) (models.Listable, error) { + ls := models.NewList() + + lh.List = ls + lh.List.SetDisplayName(ptr.To(listName)) + + return lh.List, lh.Err +} diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index c3a85ade3d..f8c308afa0 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -8,11 +8,9 @@ import ( "runtime/trace" "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" - "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/m365/collection/drive" @@ -42,6 +40,7 @@ func ConsumeRestoreCollections( ) (*support.ControllerOperationStatus, error) { var ( lrh = drive.NewSiteRestoreHandler(ac, rcc.Selector.PathService()) + listsRh = NewListsRestoreHandler(rcc.ProtectedResource.ID(), ac.Lists()) restoreMetrics support.CollectionMetrics caches = drive.NewRestoreCaches(backupDriveIDNames) el = errs.Local() @@ -89,7 +88,7 @@ func ConsumeRestoreCollections( case path.ListsCategory: metrics, err = RestoreListCollection( ictx, - ac.Stable, + listsRh, dc, rcc.RestoreConfig.Location, deets, @@ -135,7 +134,7 @@ func ConsumeRestoreCollections( // Restored List can be verified within the Site contents. func restoreListItem( ctx context.Context, - service graph.Servicer, + rh restoreHandler, itemData data.Item, siteID, destName string, ) (details.ItemInfo, error) { @@ -149,65 +148,26 @@ func restoreListItem( listName = itemData.ID() ) - byteArray, err := io.ReadAll(itemData.ToReader()) + bytes, err := io.ReadAll(itemData.ToReader()) if err != nil { return dii, clues.WrapWC(ctx, err, "reading backup data") } - oldList, err := betaAPI.CreateListFromBytes(byteArray) - if err != nil { - return dii, clues.WrapWC(ctx, err, "creating item") - } - - if name, ok := ptr.ValOK(oldList.GetDisplayName()); ok { - listName = name - } + newName := fmt.Sprintf("%s_%s", destName, listName) - var ( - newName = fmt.Sprintf("%s_%s", destName, listName) - newList = betaAPI.ToListable(oldList, newName) - contents = make([]models.ListItemable, 0) - ) - - for _, itm := range oldList.GetItems() { - temp := betaAPI.CloneListItem(itm) - contents = append(contents, temp) - } - - newList.SetItems(contents) - - // Restore to List base to M365 back store - restoredList, err := service.Client().Sites().BySiteId(siteID).Lists().Post(ctx, newList, nil) + restoredList, err := rh.PostList(ctx, newName, bytes) if err != nil { - return dii, graph.Wrap(ctx, err, "restoring list") - } - - // Uploading of ListItems is conducted after the List is restored - // Reference: https://learn.microsoft.com/en-us/graph/api/listitem-create?view=graph-rest-1.0&tabs=http - if len(contents) > 0 { - for _, lItem := range contents { - _, err := service.Client(). - Sites(). - BySiteId(siteID). - Lists(). - ByListId(ptr.Val(restoredList.GetId())). - Items(). - Post(ctx, lItem, nil) - if err != nil { - return dii, graph.Wrap(ctx, err, "restoring list items"). - With("restored_list_id", ptr.Val(restoredList.GetId())) - } - } + return dii, clues.WrapWC(ctx, err, "restoring lists") } - dii.SharePoint = ListToSPInfo(restoredList, int64(len(byteArray))) + dii.SharePoint = ListToSPInfo(restoredList, int64(len(bytes))) return dii, nil } func RestoreListCollection( ctx context.Context, - service graph.Servicer, + rh restoreHandler, dc data.RestoreCollection, restoreContainerName string, deets *details.Builder, @@ -243,7 +203,7 @@ func RestoreListCollection( itemInfo, err := restoreListItem( ctx, - service, + rh, itemData, siteID, restoreContainerName) diff --git a/src/internal/m365/service/sharepoint/api/pages.go b/src/internal/m365/service/sharepoint/api/pages.go index 968ca6c563..32b8748a57 100644 --- a/src/internal/m365/service/sharepoint/api/pages.go +++ b/src/internal/m365/service/sharepoint/api/pages.go @@ -189,7 +189,7 @@ func RestoreSitePage( } // Hydrate Page - page, err := CreatePageFromBytes(byteArray) + page, err := BytesToSitePageable(byteArray) if err != nil { return dii, clues.WrapWC(ctx, err, "creating Page object") } @@ -257,3 +257,14 @@ func PageInfo(page betamodels.SitePageable, size int64) *details.SharePointInfo Size: size, } } + +func BytesToSitePageable(bytes []byte) (betamodels.SitePageable, error) { + parsable, err := createFromBytes(bytes, betamodels.CreateSitePageFromDiscriminatorValue) + if err != nil { + return nil, clues.Wrap(err, "deserializing bytes to sharepoint page") + } + + page := parsable.(betamodels.SitePageable) + + return page, nil +} diff --git a/src/internal/m365/service/sharepoint/api/pages_test.go b/src/internal/m365/service/sharepoint/api/pages_test.go index 0f9bb20afc..0fdfbfb216 100644 --- a/src/internal/m365/service/sharepoint/api/pages_test.go +++ b/src/internal/m365/service/sharepoint/api/pages_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/alcionai/clues" + kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -21,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" + bmodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService { @@ -34,6 +36,76 @@ func createTestBetaService(t *testing.T, credentials account.M365Config) *api.Be return api.NewBetaService(adapter) } +type SharepointPageUnitSuite struct { + tester.Suite +} + +func TestSharepointPageUnitSuite(t *testing.T) { + suite.Run(t, &SharepointPageUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharepointPageUnitSuite) TestCreatePageFromBytes() { + tests := []struct { + name string + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + getBytes func(t *testing.T) []byte + }{ + { + "empty bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return make([]byte, 0) + }, + }, + { + "invalid bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return []byte("snarf") + }, + }, + { + "Valid Page", + assert.NoError, + assert.NotNil, + func(t *testing.T) []byte { + pg := bmodels.NewSitePage() + title := "Tested" + pg.SetTitle(&title) + pg.SetName(&title) + pg.SetWebUrl(&title) + + writer := kioser.NewJsonSerializationWriter() + err := writer.WriteObjectValue("", pg) + require.NoError(t, err, clues.ToCore(err)) + + byteArray, err := writer.GetSerializedContent() + require.NoError(t, err, clues.ToCore(err)) + + return byteArray + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + result, err := api.BytesToSitePageable(test.getBytes(t)) + test.checkError(t, err) + test.isNil(t, result) + if result != nil { + assert.Equal(t, "Tested", *result.GetName(), "name") + assert.Equal(t, "Tested", *result.GetTitle(), "title") + assert.Equal(t, "Tested", *result.GetWebUrl(), "webURL") + } + }) + } +} + type SharePointPageSuite struct { tester.Suite siteID string diff --git a/src/internal/m365/service/sharepoint/api/serialization.go b/src/internal/m365/service/sharepoint/api/serialization.go index c125e00571..2410ca0909 100644 --- a/src/internal/m365/service/sharepoint/api/serialization.go +++ b/src/internal/m365/service/sharepoint/api/serialization.go @@ -1,15 +1,9 @@ package api import ( - "strings" - "github.com/alcionai/clues" "github.com/microsoft/kiota-abstractions-go/serialization" kjson "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - betamodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) // createFromBytes generates an m365 object form bytes. @@ -29,184 +23,3 @@ func createFromBytes( return v, nil } - -func CreateListFromBytes(bytes []byte) (models.Listable, error) { - parsable, err := createFromBytes(bytes, models.CreateListFromDiscriminatorValue) - if err != nil { - return nil, clues.Wrap(err, "deserializing bytes to sharepoint list") - } - - list := parsable.(models.Listable) - - return list, nil -} - -func CreatePageFromBytes(bytes []byte) (betamodels.SitePageable, error) { - parsable, err := createFromBytes(bytes, betamodels.CreateSitePageFromDiscriminatorValue) - if err != nil { - return nil, clues.Wrap(err, "deserializing bytes to sharepoint page") - } - - page := parsable.(betamodels.SitePageable) - - return page, nil -} - -// ToListable utility function to encapsulate stored data for restoration. -// New Listable omits trackable fields such as `id` or `ETag` and other read-only -// objects that are prevented upon upload. Additionally, read-Only columns are -// not attached in this method. -// ListItems are not included in creation of new list, and have to be restored -// in separate call. -func ToListable(orig models.Listable, displayName string) models.Listable { - newList := models.NewList() - - newList.SetContentTypes(orig.GetContentTypes()) - newList.SetCreatedBy(orig.GetCreatedBy()) - newList.SetCreatedByUser(orig.GetCreatedByUser()) - newList.SetCreatedDateTime(orig.GetCreatedDateTime()) - newList.SetDescription(orig.GetDescription()) - newList.SetDisplayName(&displayName) - newList.SetLastModifiedBy(orig.GetLastModifiedBy()) - newList.SetLastModifiedByUser(orig.GetLastModifiedByUser()) - newList.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) - newList.SetList(orig.GetList()) - newList.SetOdataType(orig.GetOdataType()) - newList.SetParentReference(orig.GetParentReference()) - - columns := make([]models.ColumnDefinitionable, 0) - leg := map[string]struct{}{ - "Attachments": {}, - "Edit": {}, - "Content Type": {}, - } - - for _, cd := range orig.GetColumns() { - var ( - displayName string - readOnly bool - ) - - if name, ok := ptr.ValOK(cd.GetDisplayName()); ok { - displayName = name - } - - if ro, ok := ptr.ValOK(cd.GetReadOnly()); ok { - readOnly = ro - } - - _, isLegacy := leg[displayName] - - // Skips columns that cannot be uploaded for models.ColumnDefinitionable: - // - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type - if readOnly || displayName == "Title" || isLegacy { - continue - } - - columns = append(columns, cloneColumnDefinitionable(cd)) - } - - newList.SetColumns(columns) - - return newList -} - -// cloneColumnDefinitionable utility function for encapsulating models.ColumnDefinitionable data -// into new object for upload. -func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDefinitionable { - newColumn := models.NewColumnDefinition() - - newColumn.SetAdditionalData(orig.GetAdditionalData()) - newColumn.SetBoolean(orig.GetBoolean()) - newColumn.SetCalculated(orig.GetCalculated()) - newColumn.SetChoice(orig.GetChoice()) - newColumn.SetColumnGroup(orig.GetColumnGroup()) - newColumn.SetContentApprovalStatus(orig.GetContentApprovalStatus()) - newColumn.SetCurrency(orig.GetCurrency()) - newColumn.SetDateTime(orig.GetDateTime()) - newColumn.SetDefaultValue(orig.GetDefaultValue()) - newColumn.SetDescription(orig.GetDescription()) - newColumn.SetDisplayName(orig.GetDisplayName()) - newColumn.SetEnforceUniqueValues(orig.GetEnforceUniqueValues()) - newColumn.SetGeolocation(orig.GetGeolocation()) - newColumn.SetHidden(orig.GetHidden()) - newColumn.SetHyperlinkOrPicture(orig.GetHyperlinkOrPicture()) - newColumn.SetIndexed(orig.GetIndexed()) - newColumn.SetIsDeletable(orig.GetIsDeletable()) - newColumn.SetIsReorderable(orig.GetIsReorderable()) - newColumn.SetIsSealed(orig.GetIsSealed()) - newColumn.SetLookup(orig.GetLookup()) - newColumn.SetName(orig.GetName()) - newColumn.SetNumber(orig.GetNumber()) - newColumn.SetOdataType(orig.GetOdataType()) - newColumn.SetPersonOrGroup(orig.GetPersonOrGroup()) - newColumn.SetPropagateChanges(orig.GetPropagateChanges()) - newColumn.SetReadOnly(orig.GetReadOnly()) - newColumn.SetRequired(orig.GetRequired()) - newColumn.SetSourceColumn(orig.GetSourceColumn()) - newColumn.SetSourceContentType(orig.GetSourceContentType()) - newColumn.SetTerm(orig.GetTerm()) - newColumn.SetText(orig.GetText()) - newColumn.SetThumbnail(orig.GetThumbnail()) - newColumn.SetTypeEscaped(orig.GetTypeEscaped()) - newColumn.SetValidation(orig.GetValidation()) - - return newColumn -} - -// CloneListItem creates a new `SharePoint.ListItem` and stores the original item's -// M365 data into it set fields. -// - https://learn.microsoft.com/en-us/graph/api/resources/listitem?view=graph-rest-1.0 -func CloneListItem(orig models.ListItemable) models.ListItemable { - newItem := models.NewListItem() - newFieldData := retrieveFieldData(orig.GetFields()) - - newItem.SetAdditionalData(orig.GetAdditionalData()) - newItem.SetAnalytics(orig.GetAnalytics()) - newItem.SetContentType(orig.GetContentType()) - newItem.SetCreatedBy(orig.GetCreatedBy()) - newItem.SetCreatedByUser(orig.GetCreatedByUser()) - newItem.SetCreatedDateTime(orig.GetCreatedDateTime()) - newItem.SetDescription(orig.GetDescription()) - // ETag cannot be carried forward - newItem.SetFields(newFieldData) - newItem.SetLastModifiedBy(orig.GetLastModifiedBy()) - newItem.SetLastModifiedByUser(orig.GetLastModifiedByUser()) - newItem.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) - newItem.SetOdataType(orig.GetOdataType()) - // parentReference and SharePointIDs cause error on upload. - // POST Command will link items to the created list. - newItem.SetVersions(orig.GetVersions()) - - return newItem -} - -// retrieveFieldData utility function to clone raw listItem data from the embedded -// additionalData map -// Further details on FieldValueSets: -// - https://learn.microsoft.com/en-us/graph/api/resources/fieldvalueset?view=graph-rest-1.0 -func retrieveFieldData(orig models.FieldValueSetable) models.FieldValueSetable { - fields := models.NewFieldValueSet() - additionalData := make(map[string]any) - fieldData := orig.GetAdditionalData() - - // M365 Book keeping values removed during new Item Creation - // Removed Values: - // -- Prefixes -> @odata.context : absolute path to previous list - // . -> @odata.etag : Embedded link to Prior M365 ID - // -- String Match: Read-Only Fields - // -> id : previous un - for key, value := range fieldData { - if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "@") || - key == "Edit" || key == "Created" || key == "Modified" || - strings.Contains(key, "LookupId") || strings.Contains(key, "ChildCount") || strings.Contains(key, "LinkTitle") { - continue - } - - additionalData[key] = value - } - - fields.SetAdditionalData(additionalData) - - return fields -} diff --git a/src/internal/m365/service/sharepoint/api/serialization_test.go b/src/internal/m365/service/sharepoint/api/serialization_test.go index c79a822847..c52e88b589 100644 --- a/src/internal/m365/service/sharepoint/api/serialization_test.go +++ b/src/internal/m365/service/sharepoint/api/serialization_test.go @@ -4,51 +4,41 @@ import ( "testing" "github.com/alcionai/clues" - kioser "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" - bmodels "github.com/alcionai/corso/src/pkg/services/m365/api/graph/betasdk/models" ) type SerializationUnitSuite struct { tester.Suite } -func TestDataSupportSuite(t *testing.T) { +func TestSerializationUnitSuite(t *testing.T) { suite.Run(t, &SerializationUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *SerializationUnitSuite) TestCreateListFromBytes() { +func (suite *SerializationUnitSuite) TestCreateFromBytes() { listBytes, err := spMock.ListBytes("DataSupportSuite") require.NoError(suite.T(), err) tests := []struct { - name string - byteArray []byte - checkError assert.ErrorAssertionFunc - isNil assert.ValueAssertionFunc + name string + byteArray []byte + parseableFunc serialization.ParsableFactory + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc }{ { - name: "empty bytes", - byteArray: make([]byte, 0), - checkError: assert.Error, - isNil: assert.Nil, - }, - { - name: "invalid bytes", - byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), - checkError: assert.Error, - isNil: assert.Nil, - }, - { - name: "Valid List", - byteArray: listBytes, - checkError: assert.NoError, - isNil: assert.NotNil, + name: "Valid List", + byteArray: listBytes, + parseableFunc: models.CreateListFromDiscriminatorValue, + checkError: assert.NoError, + isNil: assert.NotNil, }, } @@ -56,71 +46,9 @@ func (suite *SerializationUnitSuite) TestCreateListFromBytes() { suite.Run(test.name, func() { t := suite.T() - result, err := CreateListFromBytes(test.byteArray) + result, err := createFromBytes(test.byteArray, test.parseableFunc) test.checkError(t, err, clues.ToCore(err)) test.isNil(t, result) }) } } - -func (suite *SerializationUnitSuite) TestCreatePageFromBytes() { - tests := []struct { - name string - checkError assert.ErrorAssertionFunc - isNil assert.ValueAssertionFunc - getBytes func(t *testing.T) []byte - }{ - { - "empty bytes", - assert.Error, - assert.Nil, - func(t *testing.T) []byte { - return make([]byte, 0) - }, - }, - { - "invalid bytes", - assert.Error, - assert.Nil, - func(t *testing.T) []byte { - return []byte("snarf") - }, - }, - { - "Valid Page", - assert.NoError, - assert.NotNil, - func(t *testing.T) []byte { - pg := bmodels.NewSitePage() - title := "Tested" - pg.SetTitle(&title) - pg.SetName(&title) - pg.SetWebUrl(&title) - - writer := kioser.NewJsonSerializationWriter() - err := writer.WriteObjectValue("", pg) - require.NoError(t, err, clues.ToCore(err)) - - byteArray, err := writer.GetSerializedContent() - require.NoError(t, err, clues.ToCore(err)) - - return byteArray - }, - }, - } - - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - result, err := CreatePageFromBytes(test.getBytes(t)) - test.checkError(t, err) - test.isNil(t, result) - if result != nil { - assert.Equal(t, "Tested", *result.GetName(), "name") - assert.Equal(t, "Tested", *result.GetTitle(), "title") - assert.Equal(t, "Tested", *result.GetWebUrl(), "webURL") - } - }) - } -} diff --git a/src/internal/m365/service/sharepoint/mock/mock_test.go b/src/internal/m365/service/sharepoint/mock/mock_test.go index 61590fb9ef..778b22bc1b 100644 --- a/src/internal/m365/service/sharepoint/mock/mock_test.go +++ b/src/internal/m365/service/sharepoint/mock/mock_test.go @@ -9,8 +9,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" + betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type MockSuite struct { @@ -39,7 +40,7 @@ func (suite *MockSuite) TestMockByteHydration() { bytes, err := writer.GetSerializedContent() require.NoError(t, err, clues.ToCore(err)) - _, err = api.CreateListFromBytes(bytes) + _, err = api.BytesToListable(bytes) return err }, @@ -49,7 +50,7 @@ func (suite *MockSuite) TestMockByteHydration() { transformation: func(t *testing.T) error { bytes, err := ListBytes(subject) require.NoError(t, err, clues.ToCore(err)) - _, err = api.CreateListFromBytes(bytes) + _, err = api.BytesToListable(bytes) return err }, }, @@ -57,7 +58,7 @@ func (suite *MockSuite) TestMockByteHydration() { name: "SharePoint: Page", transformation: func(t *testing.T) error { bytes := Page(subject) - _, err := api.CreatePageFromBytes(bytes) + _, err := betaAPI.BytesToSitePageable(bytes) return err }, diff --git a/src/internal/m365/service/sharepoint/restore.go b/src/internal/m365/service/sharepoint/restore.go index ab26728af8..dbe71da318 100644 --- a/src/internal/m365/service/sharepoint/restore.go +++ b/src/internal/m365/service/sharepoint/restore.go @@ -48,15 +48,13 @@ func (h *sharepointHandler) ConsumeRestoreCollections( lrh = drive.NewSiteRestoreHandler( h.apiClient, rcc.Selector.PathService()) + listsRh = site.NewListsRestoreHandler( + rcc.ProtectedResource.ID(), + h.apiClient.Lists()) restoreMetrics support.CollectionMetrics - caches = drive.NewRestoreCaches(h.backupDriveIDNames) - el = errs.Local() - ) - err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) - if err != nil { - return nil, nil, clues.Wrap(err, "initializing restore caches") - } + el = errs.Local() + ) // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. @@ -81,6 +79,13 @@ func (h *sharepointHandler) ConsumeRestoreCollections( switch dc.FullPath().Category() { case path.LibrariesCategory: + caches := drive.NewRestoreCaches(h.backupDriveIDNames) + + err = caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) + if err != nil { + return nil, nil, clues.Wrap(err, "initializing restore caches") + } + metrics, err = drive.RestoreCollection( ictx, lrh, @@ -95,7 +100,7 @@ func (h *sharepointHandler) ConsumeRestoreCollections( case path.ListsCategory: metrics, err = site.RestoreListCollection( ictx, - h.apiClient.Stable, + listsRh, dc, rcc.RestoreConfig.Location, deets, diff --git a/src/internal/m365/service/sharepoint/restore_test.go b/src/internal/m365/service/sharepoint/restore_test.go new file mode 100644 index 0000000000..18f6f41848 --- /dev/null +++ b/src/internal/m365/service/sharepoint/restore_test.go @@ -0,0 +1,63 @@ +package sharepoint + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/data/mock" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type SharepointRestoreUnitSuite struct { + tester.Suite +} + +func TestSharepointRestoreUnitSuite(t *testing.T) { + suite.Run(t, &SharepointRestoreUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharepointRestoreUnitSuite) TestSharePointHandler_ConsumeRestoreCollections_noErrorOnLists() { + t := suite.T() + siteID := "site-id" + + ctx, flush := tester.NewContext(t) + defer flush() + + pr := idname.NewProvider(siteID, siteID) + rcc := inject.RestoreConsumerConfig{ + ProtectedResource: pr, + } + pth, err := path.Builder{}. + Append("lists"). + ToDataLayerPath( + "tenant", + siteID, + path.SharePointService, + path.ListsCategory, + false) + require.NoError(t, err, clues.ToCore(err)) + + dcs := []data.RestoreCollection{ + mock.Collection{Path: pth}, + } + + sh := NewSharePointHandler(control.DefaultOptions(), api.Client{}, nil) + + _, _, err = sh.ConsumeRestoreCollections( + ctx, + rcc, + dcs, + fault.New(false), + nil) + require.NoError(t, err, "Sharepoint lists restore") +} diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go index 1225e375b9..6a4ed7eba1 100644 --- a/src/pkg/services/m365/api/lists.go +++ b/src/pkg/services/m365/api/lists.go @@ -2,14 +2,81 @@ package api import ( "context" + "strings" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/keys" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) +const ( + AttachmentsColumnName = "Attachments" + EditColumnName = "Edit" + ContentTypeColumnName = "ContentType" + CreatedColumnName = "Created" + ModifiedColumnName = "Modified" + AuthorLookupIDColumnName = "AuthorLookupId" + EditorLookupIDColumnName = "EditorLookupId" + + ContentTypeColumnDisplayName = "Content Type" + + AddressFieldName = "address" + CoordinatesFieldName = "coordinates" + DisplayNameFieldName = "displayName" + LocationURIFieldName = "locationUri" + UniqueIDFieldName = "uniqueId" + + CountryOrRegionFieldName = "CountryOrRegion" + StateFieldName = "State" + CityFieldName = "City" + PostalCodeFieldName = "PostalCode" + StreetFieldName = "Street" + GeoLocFieldName = "GeoLoc" + DispNameFieldName = "DispName" + LinkTitleFieldNamePart = "LinkTitle" + ChildCountFieldNamePart = "ChildCount" + + ReadOnlyOrHiddenFieldNamePrefix = "_" + DescoratorFieldNamePrefix = "@" +) + +var addressFieldNames = []string{ + AddressFieldName, + CoordinatesFieldName, + DisplayNameFieldName, + LocationURIFieldName, + UniqueIDFieldName, +} + +var readOnlyAddressFieldNames = []string{ + CountryOrRegionFieldName, + StateFieldName, + CityFieldName, + PostalCodeFieldName, + StreetFieldName, + GeoLocFieldName, + DispNameFieldName, +} + +var legacyColumns = keys.Set{ + AttachmentsColumnName: {}, + EditColumnName: {}, + ContentTypeColumnDisplayName: {}, +} + +var readOnlyFieldNames = keys.Set{ + AttachmentsColumnName: {}, + EditColumnName: {}, + ContentTypeColumnName: {}, + CreatedColumnName: {}, + ModifiedColumnName: {}, + AuthorLookupIDColumnName: {}, + EditorLookupIDColumnName: {}, +} + // --------------------------------------------------------------------------- // controller // --------------------------------------------------------------------------- @@ -151,6 +218,353 @@ func (c Lists) getListContents(ctx context.Context, siteID, listID string) ( return cols, cTypes, lItems, nil } +func (c Lists) PostList( + ctx context.Context, + siteID string, + listName string, + oldListByteArray []byte, +) (models.Listable, error) { + newListName := listName + + oldList, err := BytesToListable(oldListByteArray) + if err != nil { + return nil, clues.WrapWC(ctx, err, "generating list from stored bytes") + } + + // the input listName is of format: destinationName_listID + // here we replace listID with displayName of list generated from stored bytes + if name, ok := ptr.ValOK(oldList.GetDisplayName()); ok { + nameParts := strings.Split(listName, "_") + if len(nameParts) > 0 { + nameParts[len(nameParts)-1] = name + newListName = strings.Join(nameParts, "_") + } + } + + // this ensure all columns, contentTypes are set to the newList + newList := ToListable(oldList, newListName) + + // Restore to List base to M365 back store + restoredList, err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + Post(ctx, newList, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "creating list") + } + + listItems := make([]models.ListItemable, 0) + + for _, itm := range oldList.GetItems() { + temp := CloneListItem(itm) + listItems = append(listItems, temp) + } + + err = c.PostListItems( + ctx, + siteID, + ptr.Val(restoredList.GetId()), + listItems) + if err == nil { + restoredList.SetItems(listItems) + return restoredList, nil + } + + // [TODO](hitesh) double check if we need to: + // 1. rollback the entire list + // 2. restore as much list items possible and add recoverables to fault bus + // rollback list creation + err = c.DeleteList(ctx, siteID, ptr.Val(restoredList.GetId())) + + return nil, graph.Wrap(ctx, err, "deleting restored list after items creation failure").OrNil() +} + +func (c Lists) PostListItems( + ctx context.Context, + siteID, listID string, + listItems []models.ListItemable, +) error { + for _, lItem := range listItems { + _, err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Items(). + Post(ctx, lItem, nil) + if err != nil { + return graph.Wrap(ctx, err, "creating item in list") + } + } + + return nil +} + +func (c Lists) DeleteList( + ctx context.Context, + siteID, listID string, +) error { + err := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists(). + ByListId(listID). + Delete(ctx, nil) + + return graph.Wrap(ctx, err, "deleting list").OrNil() +} + +func BytesToListable(bytes []byte) (models.Listable, error) { + parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue) + if err != nil { + return nil, clues.Wrap(err, "deserializing bytes to sharepoint list") + } + + list := parsable.(models.Listable) + + return list, nil +} + +// ToListable utility function to encapsulate stored data for restoration. +// New Listable omits trackable fields such as `id` or `ETag` and other read-only +// objects that are prevented upon upload. Additionally, read-Only columns are +// not attached in this method. +// ListItems are not included in creation of new list, and have to be restored +// in separate call. +func ToListable(orig models.Listable, displayName string) models.Listable { + newList := models.NewList() + + newList.SetContentTypes(orig.GetContentTypes()) + newList.SetCreatedBy(orig.GetCreatedBy()) + newList.SetCreatedByUser(orig.GetCreatedByUser()) + newList.SetCreatedDateTime(orig.GetCreatedDateTime()) + newList.SetDescription(orig.GetDescription()) + newList.SetDisplayName(&displayName) + newList.SetLastModifiedBy(orig.GetLastModifiedBy()) + newList.SetLastModifiedByUser(orig.GetLastModifiedByUser()) + newList.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) + newList.SetList(orig.GetList()) + newList.SetOdataType(orig.GetOdataType()) + newList.SetParentReference(orig.GetParentReference()) + + columns := make([]models.ColumnDefinitionable, 0) + + for _, cd := range orig.GetColumns() { + var ( + displayName string + readOnly bool + ) + + if name, ok := ptr.ValOK(cd.GetDisplayName()); ok { + displayName = name + } + + if ro, ok := ptr.ValOK(cd.GetReadOnly()); ok { + readOnly = ro + } + + // Skips columns that cannot be uploaded for models.ColumnDefinitionable: + // - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type + if readOnly || displayName == "Title" || legacyColumns.HasKey(displayName) { + continue + } + + columns = append(columns, cloneColumnDefinitionable(cd)) + } + + newList.SetColumns(columns) + + return newList +} + +// cloneColumnDefinitionable utility function for encapsulating models.ColumnDefinitionable data +// into new object for upload. +func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDefinitionable { + newColumn := models.NewColumnDefinition() + + // column attributes + newColumn.SetName(orig.GetName()) + newColumn.SetOdataType(orig.GetOdataType()) + newColumn.SetPropagateChanges(orig.GetPropagateChanges()) + newColumn.SetReadOnly(orig.GetReadOnly()) + newColumn.SetRequired(orig.GetRequired()) + newColumn.SetAdditionalData(orig.GetAdditionalData()) + newColumn.SetDescription(orig.GetDescription()) + newColumn.SetDisplayName(orig.GetDisplayName()) + newColumn.SetSourceColumn(orig.GetSourceColumn()) + newColumn.SetSourceContentType(orig.GetSourceContentType()) + newColumn.SetHidden(orig.GetHidden()) + newColumn.SetIndexed(orig.GetIndexed()) + newColumn.SetIsDeletable(orig.GetIsDeletable()) + newColumn.SetIsReorderable(orig.GetIsReorderable()) + newColumn.SetIsSealed(orig.GetIsSealed()) + newColumn.SetTypeEscaped(orig.GetTypeEscaped()) + newColumn.SetColumnGroup(orig.GetColumnGroup()) + newColumn.SetEnforceUniqueValues(orig.GetEnforceUniqueValues()) + + // column types + setColumnType(newColumn, orig) + + // Requires nil checks to avoid Graph error: 'General exception while processing' + defaultValue := orig.GetDefaultValue() + if defaultValue != nil { + newColumn.SetDefaultValue(defaultValue) + } + + validation := orig.GetValidation() + if validation != nil { + newColumn.SetValidation(validation) + } + + return newColumn +} + +func setColumnType(newColumn *models.ColumnDefinition, orig models.ColumnDefinitionable) { + switch { + case orig.GetText() != nil: + newColumn.SetText(orig.GetText()) + case orig.GetBoolean() != nil: + newColumn.SetBoolean(orig.GetBoolean()) + case orig.GetCalculated() != nil: + newColumn.SetCalculated(orig.GetCalculated()) + case orig.GetChoice() != nil: + newColumn.SetChoice(orig.GetChoice()) + case orig.GetContentApprovalStatus() != nil: + newColumn.SetContentApprovalStatus(orig.GetContentApprovalStatus()) + case orig.GetCurrency() != nil: + newColumn.SetCurrency(orig.GetCurrency()) + case orig.GetDateTime() != nil: + newColumn.SetDateTime(orig.GetDateTime()) + case orig.GetGeolocation() != nil: + newColumn.SetGeolocation(orig.GetGeolocation()) + case orig.GetHyperlinkOrPicture() != nil: + newColumn.SetHyperlinkOrPicture(orig.GetHyperlinkOrPicture()) + case orig.GetNumber() != nil: + newColumn.SetNumber(orig.GetNumber()) + case orig.GetLookup() != nil: + newColumn.SetLookup(orig.GetLookup()) + case orig.GetThumbnail() != nil: + newColumn.SetThumbnail(orig.GetThumbnail()) + case orig.GetTerm() != nil: + newColumn.SetTerm(orig.GetTerm()) + case orig.GetPersonOrGroup() != nil: + newColumn.SetPersonOrGroup(orig.GetPersonOrGroup()) + default: + newColumn.SetText(models.NewTextColumn()) + } +} + +// CloneListItem creates a new `SharePoint.ListItem` and stores the original item's +// M365 data into it set fields. +// - https://learn.microsoft.com/en-us/graph/api/resources/listitem?view=graph-rest-1.0 +func CloneListItem(orig models.ListItemable) models.ListItemable { + newItem := models.NewListItem() + + // list item data + newFieldData := retrieveFieldData(orig.GetFields()) + newItem.SetFields(newFieldData) + + // list item attributes + newItem.SetAdditionalData(orig.GetAdditionalData()) + newItem.SetDescription(orig.GetDescription()) + newItem.SetCreatedBy(orig.GetCreatedBy()) + newItem.SetCreatedDateTime(orig.GetCreatedDateTime()) + newItem.SetLastModifiedBy(orig.GetLastModifiedBy()) + newItem.SetLastModifiedDateTime(orig.GetLastModifiedDateTime()) + newItem.SetOdataType(orig.GetOdataType()) + newItem.SetAnalytics(orig.GetAnalytics()) + newItem.SetContentType(orig.GetContentType()) + newItem.SetVersions(orig.GetVersions()) + + // Requires nil checks to avoid Graph error: 'Invalid request' + lastCreatedByUser := orig.GetCreatedByUser() + if lastCreatedByUser != nil { + newItem.SetCreatedByUser(lastCreatedByUser) + } + + lastModifiedByUser := orig.GetLastModifiedByUser() + if lastCreatedByUser != nil { + newItem.SetLastModifiedByUser(lastModifiedByUser) + } + + return newItem +} + +// retrieveFieldData utility function to clone raw listItem data from the embedded +// additionalData map +// Further details on FieldValueSets: +// - https://learn.microsoft.com/en-us/graph/api/resources/fieldvalueset?view=graph-rest-1.0 +func retrieveFieldData(orig models.FieldValueSetable) models.FieldValueSetable { + fields := models.NewFieldValueSet() + additionalData := filterAdditionalData(orig) + + retainPrimaryAddressField(additionalData) + + fields.SetAdditionalData(additionalData) + + return fields +} + +func filterAdditionalData(orig models.FieldValueSetable) map[string]any { + if orig == nil { + return make(map[string]any) + } + + fieldData := orig.GetAdditionalData() + filteredData := make(map[string]any) + + for key, value := range fieldData { + if shouldFilterField(key, value) { + continue + } + + filteredData[key] = value + } + + return filteredData +} + +func shouldFilterField(key string, value any) bool { + return readOnlyFieldNames.HasKey(key) || + strings.HasPrefix(key, ReadOnlyOrHiddenFieldNamePrefix) || + strings.HasPrefix(key, DescoratorFieldNamePrefix) || + strings.Contains(key, LinkTitleFieldNamePart) || + strings.Contains(key, ChildCountFieldNamePart) +} + +func retainPrimaryAddressField(additionalData map[string]any) { + if !hasAddressFields(additionalData) { + return + } + + for _, k := range readOnlyAddressFieldNames { + delete(additionalData, k) + } +} + +func hasAddressFields(additionalData map[string]any) bool { + if !keys.HasKeys(additionalData, readOnlyAddressFieldNames...) { + return false + } + + for _, value := range additionalData { + nestedFields, ok := value.(map[string]any) + if !ok || keys.HasKeys(nestedFields, GeoLocFieldName) { + continue + } + + if keys.HasKeys(nestedFields, addressFieldNames...) { + return true + } + } + + return false +} + func (c Lists) getListItemFields( ctx context.Context, siteID, listID, itemID string, diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go index b81ecc9d1a..6cd7c37e80 100644 --- a/src/pkg/services/m365/api/lists_test.go +++ b/src/pkg/services/m365/api/lists_test.go @@ -5,12 +5,14 @@ import ( "github.com/alcionai/clues" "github.com/h2non/gock" + kjson "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" + spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/control/testdata" @@ -18,6 +20,429 @@ import ( graphTD "github.com/alcionai/corso/src/pkg/services/m365/api/graph/testdata" ) +type ListsUnitSuite struct { + tester.Suite +} + +func TestListsUnitSuite(t *testing.T) { + suite.Run(t, &ListsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ListsUnitSuite) TestBytesToListable() { + listBytes, err := spMock.ListBytes("DataSupportSuite") + require.NoError(suite.T(), err) + + tests := []struct { + name string + byteArray []byte + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + }{ + { + name: "empty bytes", + byteArray: make([]byte, 0), + checkError: assert.Error, + isNil: assert.Nil, + }, + { + name: "invalid bytes", + byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), + checkError: assert.Error, + isNil: assert.Nil, + }, + { + name: "Valid List", + byteArray: listBytes, + checkError: assert.NoError, + isNil: assert.NotNil, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + result, err := BytesToListable(test.byteArray) + test.checkError(t, err, clues.ToCore(err)) + test.isNil(t, result) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_GetValidation() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + expect assert.ValueAssertionFunc + }{ + { + name: "column validation not set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + + return cd + }, + expect: assert.Nil, + }, + { + name: "column validation set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + colValidation := models.NewColumnValidation() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + cd.SetValidation(colValidation) + + return cd + }, + expect: assert.NotNil, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + + test.expect(t, newCd.GetValidation()) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_GetDefaultValue() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + expect func(t *testing.T, cd models.ColumnDefinitionable) + }{ + { + name: "column default value not set", + getOrig: func() models.ColumnDefinitionable { + textColumn := models.NewTextColumn() + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + + return cd + }, + expect: func(t *testing.T, cd models.ColumnDefinitionable) { + assert.Nil(t, cd.GetDefaultValue()) + }, + }, + { + name: "column default value set", + getOrig: func() models.ColumnDefinitionable { + defaultVal := "some-val" + + textColumn := models.NewTextColumn() + + colDefaultVal := models.NewDefaultColumnValue() + colDefaultVal.SetValue(ptr.To(defaultVal)) + + cd := models.NewColumnDefinition() + cd.SetText(textColumn) + cd.SetDefaultValue(colDefaultVal) + + return cd + }, + expect: func(t *testing.T, cd models.ColumnDefinitionable) { + assert.NotNil(t, cd.GetDefaultValue()) + assert.Equal(t, "some-val", ptr.Val(cd.GetDefaultValue().GetValue())) + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + test.expect(t, newCd) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_ColumnType() { + tests := []struct { + name string + getOrig func() models.ColumnDefinitionable + checkFn func(models.ColumnDefinitionable) bool + }{ + { + name: "column type should be number", + getOrig: func() models.ColumnDefinitionable { + numColumn := models.NewNumberColumn() + + cd := models.NewColumnDefinition() + cd.SetNumber(numColumn) + + return cd + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetNumber() != nil + }, + }, + { + name: "column type should be person or group", + getOrig: func() models.ColumnDefinitionable { + pgColumn := models.NewPersonOrGroupColumn() + + cd := models.NewColumnDefinition() + cd.SetPersonOrGroup(pgColumn) + + return cd + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetPersonOrGroup() != nil + }, + }, + { + name: "column type should default to text", + getOrig: func() models.ColumnDefinitionable { + return models.NewColumnDefinition() + }, + checkFn: func(cd models.ColumnDefinitionable) bool { + return cd.GetText() != nil + }, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + orig := test.getOrig() + newCd := cloneColumnDefinitionable(orig) + + require.NotEmpty(t, newCd) + assert.True(t, test.checkFn(newCd)) + }) + } +} + +func (suite *ListsUnitSuite) TestColumnDefinitionable_LegacyColumns() { + listName := "test-list" + textColumnName := "ItemName" + textColumnDisplayName := "Item Name" + titleColumnName := "Title" + titleColumnDisplayName := "Title" + readOnlyColumnName := "TestColumn" + readOnlyColumnDisplayName := "Test Column" + + contentTypeCd := models.NewColumnDefinition() + contentTypeCd.SetName(ptr.To(ContentTypeColumnName)) + contentTypeCd.SetDisplayName(ptr.To(ContentTypeColumnDisplayName)) + + attachmentCd := models.NewColumnDefinition() + attachmentCd.SetName(ptr.To(AttachmentsColumnName)) + attachmentCd.SetDisplayName(ptr.To(AttachmentsColumnName)) + + editCd := models.NewColumnDefinition() + editCd.SetName(ptr.To(EditColumnName)) + editCd.SetDisplayName(ptr.To(EditColumnName)) + + textCol := models.NewTextColumn() + titleCol := models.NewTextColumn() + roCol := models.NewTextColumn() + + textCd := models.NewColumnDefinition() + textCd.SetName(ptr.To(textColumnName)) + textCd.SetDisplayName(ptr.To(textColumnDisplayName)) + textCd.SetText(textCol) + + titleCd := models.NewColumnDefinition() + titleCd.SetName(ptr.To(titleColumnName)) + titleCd.SetDisplayName(ptr.To(titleColumnDisplayName)) + titleCd.SetText(titleCol) + + roCd := models.NewColumnDefinition() + roCd.SetName(ptr.To(readOnlyColumnName)) + roCd.SetDisplayName(ptr.To(readOnlyColumnDisplayName)) + roCd.SetText(roCol) + roCd.SetReadOnly(ptr.To(true)) + + tests := []struct { + name string + getList func() *models.List + length int + }{ + { + name: "all legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + }) + return lst + }, + length: 0, + }, + { + name: "title and legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + titleCd, + }) + return lst + }, + length: 0, + }, + { + name: "readonly and legacy columns", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + roCd, + }) + return lst + }, + length: 0, + }, + { + name: "legacy and a text column", + getList: func() *models.List { + lst := models.NewList() + lst.SetColumns([]models.ColumnDefinitionable{ + contentTypeCd, + attachmentCd, + editCd, + textCd, + }) + return lst + }, + length: 1, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + clonedList := ToListable(test.getList(), listName) + require.NotEmpty(t, clonedList) + + cols := clonedList.GetColumns() + assert.Len(t, cols, test.length) + }) + } +} + +func (suite *ListsUnitSuite) TestFieldValueSetable() { + t := suite.T() + + additionalData := map[string]any{ + DescoratorFieldNamePrefix + "odata.etag": "14fe12b2-e180-49f7-8fc3-5936f3dcf5d2,1", + ReadOnlyOrHiddenFieldNamePrefix + "UIVersionString": "1.0", + AuthorLookupIDColumnName: "6", + EditorLookupIDColumnName: "6", + "Item" + ChildCountFieldNamePart: "0", + "Folder" + ChildCountFieldNamePart: "0", + ModifiedColumnName: "2023-12-13T15:47:51Z", + CreatedColumnName: "2023-12-13T15:47:51Z", + EditColumnName: "", + LinkTitleFieldNamePart + "NoMenu": "Person1", + } + + origFs := models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs := retrieveFieldData(origFs) + fsAdditionalData := fs.GetAdditionalData() + assert.Empty(t, fsAdditionalData) + + additionalData["itemName"] = "item-1" + origFs = models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs = retrieveFieldData(origFs) + fsAdditionalData = fs.GetAdditionalData() + assert.NotEmpty(t, fsAdditionalData) + + val, ok := fsAdditionalData["itemName"] + assert.True(t, ok) + assert.Equal(t, "item-1", val) +} + +func (suite *ListsUnitSuite) TestFieldValueSetable_Location() { + t := suite.T() + + additionalData := map[string]any{ + "MyAddress": map[string]any{ + AddressFieldName: map[string]any{ + "city": "Tagaytay", + "countryOrRegion": "Philippines", + "postalCode": "4120", + "state": "Calabarzon", + "street": "Prime Residences CityLand 1852", + }, + CoordinatesFieldName: map[string]any{ + "latitude": "14.1153", + "longitude": "120.962", + }, + DisplayNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + LocationURIFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + UniqueIDFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + }, + CountryOrRegionFieldName: "Philippines", + StateFieldName: "Calabarzon", + CityFieldName: "Tagaytay", + PostalCodeFieldName: "4120", + StreetFieldName: "Prime Residences CityLand 1852", + GeoLocFieldName: map[string]any{ + "latitude": 14.1153, + "longitude": 120.962, + }, + DispNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + } + + expectedData := map[string]any{ + "MyAddress": map[string]any{ + AddressFieldName: map[string]any{ + "city": "Tagaytay", + "countryOrRegion": "Philippines", + "postalCode": "4120", + "state": "Calabarzon", + "street": "Prime Residences CityLand 1852", + }, + CoordinatesFieldName: map[string]any{ + "latitude": "14.1153", + "longitude": "120.962", + }, + DisplayNameFieldName: "B123 Unit 1852 Prime Residences Tagaytay", + LocationURIFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + UniqueIDFieldName: "https://www.bingapis.com/api/v6/localbusinesses/YN8144x496766267081923032", + }, + } + + origFs := models.NewFieldValueSet() + origFs.SetAdditionalData(additionalData) + + fs := retrieveFieldData(origFs) + fsAdditionalData := fs.GetAdditionalData() + assert.Equal(t, expectedData, fsAdditionalData) +} + type ListsAPIIntgSuite struct { tester.Suite its intgTesterSetup @@ -217,3 +642,117 @@ func (suite *ListsAPIIntgSuite) TestLists_GetListByID() { }) } } + +func (suite *ListsAPIIntgSuite) TestLists_PostList() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + fieldsData, list := getFieldsDataAndList() + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + newList, err := acl.PostList(ctx, siteID, listName, oldListByteArray) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, listName, ptr.Val(newList.GetDisplayName())) + + _, err = acl.PostList(ctx, siteID, listName, oldListByteArray) + require.Error(t, err) + + newListItems := newList.GetItems() + require.Less(t, 0, len(newListItems)) + + newListItemFields := newListItems[0].GetFields() + require.NotEmpty(t, newListItemFields) + + newListItemsData := newListItemFields.GetAdditionalData() + require.NotEmpty(t, newListItemsData) + + for k, v := range newListItemsData { + assert.Equal(t, fieldsData[k], ptr.Val(v.(*string))) + } + + err = acl.DeleteList(ctx, siteID, ptr.Val(newList.GetId())) + require.NoError(t, err) +} + +func (suite *ListsAPIIntgSuite) TestLists_DeleteList() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + siteID = suite.its.site.id + listName = testdata.DefaultRestoreConfig("list_api_post_list").Location + ) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + _, list := getFieldsDataAndList() + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + oldListByteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + newList, err := acl.PostList(ctx, siteID, listName, oldListByteArray) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, listName, ptr.Val(newList.GetDisplayName())) + + err = acl.DeleteList(ctx, siteID, ptr.Val(newList.GetId())) + require.NoError(t, err) +} + +func getFieldsDataAndList() (map[string]any, *models.List) { + oldListID := "old-list" + listItemID := "list-item1" + textColumnDefID := "list-col1" + textColumnDefName := "itemName" + template := "genericList" + + listInfo := models.NewListInfo() + listInfo.SetTemplate(&template) + + textColumn := models.NewTextColumn() + + txtColumnDef := models.NewColumnDefinition() + txtColumnDef.SetId(&textColumnDefID) + txtColumnDef.SetName(&textColumnDefName) + txtColumnDef.SetText(textColumn) + + fields := models.NewFieldValueSet() + fieldsData := map[string]any{ + textColumnDefName: "item1", + } + fields.SetAdditionalData(fieldsData) + + listItem := models.NewListItem() + listItem.SetId(&listItemID) + listItem.SetFields(fields) + + list := models.NewList() + list.SetId(&oldListID) + list.SetList(listInfo) + list.SetColumns([]models.ColumnDefinitionable{txtColumnDef}) + list.SetItems([]models.ListItemable{listItem}) + + return fieldsData, list +}