lists pager implementation (#4815)

Pager implementation for lists and associated relationships

#### Does this PR need a docs update or release note?
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

#### Issue(s)
#4754 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/m365/collection/site/backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/alcionai/clues"
 
 	"github.com/alcionai/corso/src/internal/common/prefixmatcher"
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/internal/m365/collection/drive"
 	betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
@@ -153,14 +154,15 @@ func CollectLists(
 	var (
 		el   = errs.Local()
 		spcs = make([]data.BackupCollection, 0)
+		acc  = api.CallConfig{Select: idAnd("displayName")}
 	)
 
-	lists, err := PreFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID())
+	lists, err := bh.GetItems(ctx, acc)
 	if err != nil {
 		return nil, err
 	}
 
-	for _, tuple := range lists {
+	for _, list := range lists {
 		if el.Failure() != nil {
 			break
 		}
@@ -171,7 +173,7 @@ func CollectLists(
 			path.SharePointService,
 			path.ListsCategory,
 			false,
-			tuple.Name)
+			ptr.Val(list.GetDisplayName()))
 		if err != nil {
 			el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating list collection path"))
 		}
@@ -183,10 +185,20 @@ func CollectLists(
 			scope,
 			su,
 			bpc.Options)
-		collection.AddJob(tuple.ID)
+		collection.AddJob(ptr.Val(list.GetId()))
 
 		spcs = append(spcs, collection)
 	}
 
 	return spcs, el.Failure()
 }
+
+func idAnd(ss ...string) []string {
+	id := []string{"id"}
+
+	if len(ss) == 0 {
+		return id
+	}
+
+	return append(id, ss...)
+}

src/internal/m365/collection/site/handlers.go

@@ -4,12 +4,19 @@ import (
 	"context"
 
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
 )
 
 type backupHandler interface {
 	getItemByIDer
+	getItemser
 }
 
 type getItemByIDer interface {
 	GetItemByID(ctx context.Context, itemID string) (models.Listable, error)
 }
+
+type getItemser interface {
+	GetItems(ctx context.Context, cc api.CallConfig) ([]models.Listable, error)
+}

src/internal/m365/collection/site/lists.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/microsoftgraph/msgraph-sdk-go/sites"
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/backup/details"
@@ -31,65 +30,6 @@ func ListToSPInfo(lst models.Listable, size int64) *details.SharePointInfo {
 	}
 }
 
-type ListTuple struct {
-	ID   string
-	Name string
-}
-
-func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration {
-	selecting := []string{"id", "displayName"}
-	queryOptions := sites.ItemListsRequestBuilderGetQueryParameters{
-		Select: selecting,
-	}
-	options := &sites.ItemListsRequestBuilderGetRequestConfiguration{
-		QueryParameters: &queryOptions,
-	}
-
-	return options
-}
-
-func PreFetchLists(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID string,
-) ([]ListTuple, error) {
-	var (
-		builder    = gs.Client().Sites().BySiteId(siteID).Lists()
-		options    = preFetchListOptions()
-		listTuples = make([]ListTuple, 0)
-	)
-
-	for {
-		resp, err := builder.Get(ctx, options)
-		if err != nil {
-			return nil, graph.Wrap(ctx, err, "getting lists")
-		}
-
-		for _, entry := range resp.GetValue() {
-			var (
-				id   = ptr.Val(entry.GetId())
-				name = ptr.Val(entry.GetDisplayName())
-				temp = ListTuple{ID: id, Name: name}
-			)
-
-			if len(name) == 0 {
-				temp.Name = id
-			}
-
-			listTuples = append(listTuples, temp)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsRequestBuilder(link, gs.Adapter())
-	}
-
-	return listTuples, nil
-}
-
 // DeleteList removes a list object from a site.
 // deletes require unique http clients
 // https://github.com/alcionai/corso/issues/2707

src/internal/m365/collection/site/lists_handler.go

@@ -25,3 +25,7 @@ func NewListsBackupHandler(protectedResource string, ac api.Lists) listsBackupHa
 func (bh listsBackupHandler) GetItemByID(ctx context.Context, itemID string) (models.Listable, error) {
 	return bh.ac.GetListByID(ctx, bh.protectedResource, itemID)
 }
+
+func (bh listsBackupHandler) GetItems(ctx context.Context, cc api.CallConfig) ([]models.Listable, error) {
+	return bh.ac.GetLists(ctx, bh.protectedResource, cc)
+}

src/pkg/services/m365/api/lists.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/alcionai/clues"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/microsoftgraph/msgraph-sdk-go/sites"
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
@@ -109,213 +108,36 @@ func (c Lists) getListContents(ctx context.Context, siteID, listID string) (
 	[]models.ListItemable,
 	error,
 ) {
-	cols, err := c.getColumns(ctx, siteID, listID, "")
+	cols, err := c.GetListColumns(ctx, siteID, listID, CallConfig{})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
-	cTypes, err := c.getContentTypes(ctx, siteID, listID)
+	cTypes, err := c.GetContentTypes(ctx, siteID, listID, CallConfig{})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
-	lItems, err := c.getListItems(ctx, siteID, listID)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	return cols, cTypes, lItems, nil
-}
-
-// getListItems utility for retrieving ListItem data and the associated relationship
-// data. Additional call append data to the tracked items, and do not create additional collections.
-// Additional Call:
-// * Fields
-// [TODO: Hitesh] Implement pager for list items
-func (c Lists) getListItems(ctx context.Context, siteID, listID string) ([]models.ListItemable, error) {
-	var (
-		prefix = c.Stable.
-			Client().
-			Sites().
-			BySiteId(siteID).
-			Lists().
-			ByListId(listID)
-		builder = prefix.Items()
-		itms    = make([]models.ListItemable, 0)
-	)
-
-	for {
-		resp, err := builder.Get(ctx, nil)
+	for i := 0; i < len(cTypes); i++ {
+		columnLinks, err := c.GetColumnLinks(ctx, siteID, listID, ptr.Val(cTypes[i].GetId()), CallConfig{})
 		if err != nil {
-			return nil, err
-		}
-
-		for _, itm := range resp.GetValue() {
-			newPrefix := prefix.Items().ByListItemId(ptr.Val(itm.GetId()))
-
-			fields, err := newPrefix.Fields().Get(ctx, nil)
-			if err != nil {
-				return nil, graph.Wrap(ctx, err, "fetching list fields")
-			}
-
-			itm.SetFields(fields)
-
-			itms = append(itms, itm)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsItemItemsRequestBuilder(link, c.Stable.Adapter())
-	}
-
-	return itms, nil
-}
-
-// getColumns utility function to return columns from a site.
-// An additional call required to check for details concerning the SourceColumn.
-// For additional details:  https://learn.microsoft.com/en-us/graph/api/resources/columndefinition?view=graph-rest-1.0
-// TODO: Refactor on if/else (dadams39)
-// [TODO: Hitesh] Implement pager for list columns
-func (c Lists) getColumns(ctx context.Context, siteID, listID, cTypeID string) ([]models.ColumnDefinitionable, error) {
-	cs := make([]models.ColumnDefinitionable, 0)
-
-	prefixBuilder := c.Stable.
-		Client().
-		Sites().
-		BySiteId(siteID).
-		Lists().
-		ByListId(listID)
-
-	if len(cTypeID) == 0 {
-		builder := prefixBuilder.Columns()
-
-		for {
-			resp, err := builder.Get(ctx, nil)
-			if err != nil {
-				return nil, graph.Wrap(ctx, err, "getting list columns")
-			}
-
-			cs = append(cs, resp.GetValue()...)
-
-			link, ok := ptr.ValOK(resp.GetOdataNextLink())
-			if !ok {
-				break
-			}
-
-			builder = sites.NewItemListsItemColumnsRequestBuilder(link, c.Stable.Adapter())
+			return nil, nil, nil, err
 		}
-	} else {
-		builder := prefixBuilder.ContentTypes().ByContentTypeId(cTypeID).Columns()
 
-		for {
-			resp, err := builder.Get(ctx, nil)
-			if err != nil {
-				return nil, graph.Wrap(ctx, err, "getting content columns")
-			}
+		cTypes[i].SetColumnLinks(columnLinks)
 
-			cs = append(cs, resp.GetValue()...)
-
-			link, ok := ptr.ValOK(resp.GetOdataNextLink())
-			if !ok {
-				break
-			}
-
-			builder = sites.NewItemListsItemContentTypesItemColumnsRequestBuilder(link, c.Stable.Adapter())
-		}
-	}
-
-	return cs, nil
-}
-
-// getContentTypes retrieves all data for content type. Additional queries required
-// for the following:
-// - ColumnLinks
-// - Columns
-// Expand queries not used to retrieve the above. Possibly more than 20.
-// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
-// [TODO: Hitesh] Implement pager for list content types
-func (c Lists) getContentTypes(ctx context.Context, siteID, listID string) ([]models.ContentTypeable, error) {
-	var (
-		cTypes  = make([]models.ContentTypeable, 0)
-		builder = c.Stable.
-			Client().
-			Sites().
-			BySiteId(siteID).
-			Lists().
-			ByListId(listID).
-			ContentTypes()
-	)
-
-	for {
-		resp, err := builder.Get(ctx, nil)
+		cTypeColumns, err := c.GetCTypesColumns(ctx, siteID, listID, ptr.Val(cTypes[i].GetId()), CallConfig{})
 		if err != nil {
-			return nil, err
-		}
-
-		for _, cont := range resp.GetValue() {
-			id := ptr.Val(cont.GetId())
-
-			links, err := c.getColumnLinks(ctx, siteID, listID, id)
-			if err != nil {
-				return nil, err
-			}
-
-			cont.SetColumnLinks(links)
-
-			cs, err := c.getColumns(ctx, siteID, listID, id)
-			if err != nil {
-				return nil, err
-			}
-
-			cont.SetColumns(cs)
-
-			cTypes = append(cTypes, cont)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
+			return nil, nil, nil, err
 		}
 
-		builder = sites.NewItemListsItemContentTypesRequestBuilder(link, c.Stable.Adapter())
+		cTypes[i].SetColumns(cTypeColumns)
 	}
 
-	return cTypes, nil
-}
-
-// [TODO: Hitesh] Implement pager for column links
-func (c Lists) getColumnLinks(ctx context.Context,
-	siteID, listID, cTypeID string,
-) ([]models.ColumnLinkable, error) {
-	var (
-		prefixBuilder = c.Stable.
-				Client().
-				Sites().
-				BySiteId(siteID).
-				Lists().
-				ByListId(listID)
-		builder = prefixBuilder.ContentTypes().ByContentTypeId(cTypeID).ColumnLinks()
-		links   = make([]models.ColumnLinkable, 0)
-	)
-
-	for {
-		resp, err := builder.Get(ctx, nil)
-		if err != nil {
-			return nil, graph.Wrap(ctx, err, "getting column links")
-		}
-
-		links = append(links, resp.GetValue()...)
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsItemContentTypesItemColumnLinksRequestBuilder(link, c.Stable.Adapter())
+	lItems, err := c.GetListItems(ctx, siteID, listID, CallConfig{})
+	if err != nil {
+		return nil, nil, nil, err
 	}
 
-	return links, nil
+	return cols, cTypes, lItems, nil
 }