Extend ToPagedList to also support count query using a separate query

docs/documents/querying/linq/paging.md

@@ -20,7 +20,7 @@ var hasPrevPage = pagedList.HasPreviousPage; // check if there is previous page
 var firstItemOnPage = pagedList.FirstItemOnPage; // one-based index of first item in current page
 var lastItemOnPage = pagedList.LastItemOnPage; // one-based index of last item in current page
 ```
-<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L216-L231' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_to_paged_list' title='Start of snippet'>anchor</a></sup>
+<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L224-L239' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_to_paged_list' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 <!-- snippet: sample_to_paged_list_async -->
@@ -31,10 +31,25 @@ var pageSize = 10;
 
 var pagedList = await theSession.Query<Target>().ToPagedListAsync(pageNumber, pageSize);
 ```
-<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L240-L245' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_to_paged_list_async' title='Start of snippet'>anchor</a></sup>
+<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L248-L253' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_to_paged_list_async' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
-If you want to create you own paged queries, just use the `Take()` and `Skip()` Linq operators in combination with `Stats()`
+For total row count, by default it internally uses `Stats()` based query which is a window function using `count(*) OVER()`. This works well for small to medium datasets but won't perform well for large dataset with millions of records. To deal with large datasets, `ToPagedList` and `ToPagedListAsync` support a method override to pass boolean `useCountQuery` as `true` which will run a separate `count(*)` query for the total rows. See an example below:
+
+<!-- snippet: sample_to_paged_list_seperate_count_query -->
+<a id='snippet-sample_to_paged_list_seperate_count_query'></a>
+```cs
+var pageNumber = 2;
+var pageSize = 10;
+var pagedList = theSession.Query<Target>().ToPagedList(pageNumber, pageSize, true);
+
+// paged list also provides a list of helper properties to deal with pagination aspects
+var totalItems = pagedList.TotalItemCount; // get total number records
+```
+<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L261-L268' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_to_paged_list_seperate_count_query' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+If you want to construct you own paged queries without using `ToPagedList`, just use the `Take()` and `Skip()` Linq operators in combination with `Stats()`
 
 <!-- snippet: sample_using-query-statistics -->
 <a id='snippet-sample_using-query-statistics'></a>
@@ -64,7 +79,7 @@ public async Task can_get_the_total_in_results()
     stats.TotalResults.ShouldBe(count);
 }
 ```
-<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L157-L183' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_using-query-statistics' title='Start of snippet'>anchor</a></sup>
+<sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Acceptance/statistics_and_paged_list.cs#L165-L191' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_using-query-statistics' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 For the sake of completeness, the SQL generated in the operation above by Marten would be:

src/LinqTests/Acceptance/statistics_and_paged_list.cs

@@ -20,18 +20,26 @@ public class PaginationTestDocument
     public string Id { get; set; }
 }
 
-public class ToPagedListData<T> : IEnumerable<object[]>
+public class ToPagedListData<T>: IEnumerable<object[]>
 {
     private static readonly Func<IQueryable<T>, int, int, Task<IPagedList<T>>> ToPagedListAsync
         = (query, pageNumber, pageSize) => query.ToPagedListAsync(pageNumber, pageSize);
 
+    private static readonly Func<IQueryable<T>, int, int, Task<IPagedList<T>>> ToPagedListWithCountQueryAsync
+        = (query, pageNumber, pageSize) => query.ToPagedListAsync(pageNumber, pageSize, true);
+
     private static readonly Func<IQueryable<T>, int, int, Task<IPagedList<T>>> ToPagedListSync
         = (query, pageNumber, pageSize) => Task.FromResult(query.ToPagedList(pageNumber, pageSize));
 
+    private static readonly Func<IQueryable<T>, int, int, Task<IPagedList<T>>> ToPagedListWithCountQuerySync
+        = (query, pageNumber, pageSize) => Task.FromResult(query.ToPagedList(pageNumber, pageSize, true));
+
     public IEnumerator<object[]> GetEnumerator()
     {
-        yield return new object []{ ToPagedListAsync };
-        yield return new object[] { ToPagedListSync };
+        yield return [ToPagedListAsync];
+        yield return [ToPagedListWithCountQueryAsync];
+        yield return [ToPagedListSync];
+        yield return [ToPagedListWithCountQuerySync];
     }
 
     IEnumerator IEnumerable.GetEnumerator()
@@ -246,6 +254,24 @@ public async Task can_return_paged_result_async()
 
         pagedList.Count.ShouldBe(pageSize);
     }
+
+    [Fact]
+    public void can_return_paged_result_using_separate_count_query()
+    {
+        #region sample_to_paged_list_seperate_count_query
+        var pageNumber = 2;
+        var pageSize = 10;
+        var pagedList = theSession.Query<Target>().ToPagedList(pageNumber, pageSize, true);
+
+        // paged list also provides a list of helper properties to deal with pagination aspects
+        var totalItems = pagedList.TotalItemCount; // get total number records
+        #endregion
+
+        pagedList.Count.ShouldBe(pageSize);
+
+    }
+
+
     [Theory]
     [ClassData(typeof(ToPagedListData<Target>))]
     public async Task invalid_pagenumber_should_throw_exception(Func<IQueryable<Target>, int, int, Task<IPagedList<Target>>> toPagedList)

src/Marten/Pagination/PagedList.cs

@@ -104,30 +104,45 @@ IEnumerator IEnumerable.GetEnumerator()
 
     /// <summary>
     ///     Static method to create a new instance of the <see cref="PagedList{T}
-    /// 
+    ///
     /// </summary>
     /// <param name="queryable"></param>
     /// <param name="pageNumber"></param>
     /// <param name="pageSize"></param>
-    public static PagedList<T> Create(IQueryable<T> queryable, int pageNumber, int pageSize)
+    public static PagedList<T> Create(IQueryable<T> queryable, int pageNumber, int pageSize, bool useCountQuery = false)
     {
         var pagedList = new PagedList<T>();
-        pagedList.Init(queryable, pageNumber, pageSize);
+        pagedList.Init(queryable, pageNumber, pageSize, useCountQuery);
         return pagedList;
     }
 
     /// <summary>
     ///     Async static method to create a new instance of the <see cref="PagedList{T}
-    /// 
+    ///
     /// </summary>
     /// <param name="queryable"></param>
     /// <param name="pageNumber"></param>
     /// <param name="pageSize"></param>
+    [Obsolete("This method is deprecated, use new method override with useCountQuery.", false)]
     public static async Task<PagedList<T>> CreateAsync(IQueryable<T> queryable, int pageNumber, int pageSize,
         CancellationToken token = default)
+    {
+        return await CreateAsync(queryable, pageNumber, pageSize, false, token).ConfigureAwait(false);
+    }
+
+    /// <summary>
+    ///     Async static method to create a new instance of the <see cref="PagedList{T}
+    ///
+    /// </summary>
+    /// <param name="queryable"></param>
+    /// <param name="pageNumber"></param>
+    /// <param name="pageSize"></param>
+    /// <param name="useCountQuery">Use a separate count query rather than using Stats.</param>
+    public static async Task<PagedList<T>> CreateAsync(IQueryable<T> queryable, int pageNumber, int pageSize,
+        bool useCountQuery, CancellationToken token = default)
     {
         var pagedList = new PagedList<T>();
-        await pagedList.InitAsync(queryable, pageNumber, pageSize, token).ConfigureAwait(false);
+        await pagedList.InitAsync(queryable, pageNumber, pageSize, useCountQuery, token).ConfigureAwait(false);
         return pagedList;
     }
 
@@ -137,12 +152,17 @@ public static async Task<PagedList<T>> CreateAsync(IQueryable<T> queryable, int
     /// <param name="queryable">Query for which data has to be fetched</param>
     /// <param name="pageSize">Page size</param>
     /// <param name="totalItemCount">Total count of all records</param>
-    public void Init(IQueryable<T> queryable, int pageNumber, int pageSize)
+    /// <param name="useCountQuery">Use a separate count query rather than using Stats. Default is false and uses Stats</param>
+    public void Init(IQueryable<T> queryable, int pageNumber, int pageSize, bool useCountQuery=false)
     {
-        var query = PrepareQuery(queryable, pageNumber, pageSize, out var statistics);
+        var query = PrepareQuery(queryable, pageNumber, pageSize, useCountQuery, out var statistics);
 
-        var items = query.ToList();
+        if (useCountQuery)
+        {
+            statistics.TotalResults = queryable.LongCount();
+        }
 
+        var items = query.ToList();
         ProcessResults(pageSize, items, statistics);
     }
 
@@ -152,17 +172,35 @@ public void Init(IQueryable<T> queryable, int pageNumber, int pageSize)
     /// <param name="queryable">Query for which data has to be fetched</param>
     /// <param name="pageSize">Page size</param>
     /// <param name="totalItemCount">Total count of all records</param>
+    [Obsolete("This method is deprecated, use new method override with useCountQuery.", false)]
     public async Task InitAsync(IQueryable<T> queryable, int pageNumber, int pageSize,
         CancellationToken token = default)
     {
-        var query = PrepareQuery(queryable, pageNumber, pageSize, out var statistics);
+        await InitAsync(queryable, pageNumber, pageSize, false, token).ConfigureAwait(false);
+    }
+
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="PagedList{T}" /> class with a override to use separate count query.
+    /// </summary>
+    /// <param name="queryable">Query for which data has to be fetched</param>
+    /// <param name="pageSize">Page size</param>
+    /// <param name="totalItemCount">Total count of all records</param>
+    /// <param name="useCountQuery">Use a separate count query rather than using Stats</param>
+    public async Task InitAsync(IQueryable<T> queryable, int pageNumber, int pageSize, bool useCountQuery,
+        CancellationToken token = default)
+    {
+        var query = PrepareQuery(queryable, pageNumber, pageSize, useCountQuery, out var statistics);
 
-        var items = await query.ToListAsync(token).ConfigureAwait(false);
+        if (useCountQuery)
+        {
+            statistics.TotalResults = await queryable.LongCountAsync(token).ConfigureAwait(false);
+        }
 
+        var items = await query.ToListAsync(token).ConfigureAwait(false);
         ProcessResults(pageSize, items, statistics);
     }
 
-    private IQueryable<T> PrepareQuery(IQueryable<T> queryable, int pageNumber, int pageSize,
+    private IQueryable<T> PrepareQuery(IQueryable<T> queryable, int pageNumber, int pageSize, bool useCountQuery,
         out QueryStatistics statistics)
     {
         // throw an argument exception if page number is less than one
@@ -180,6 +218,15 @@ private IQueryable<T> PrepareQuery(IQueryable<T> queryable, int pageNumber, int
         PageNumber = pageNumber;
         PageSize = pageSize;
 
+        if (useCountQuery)
+        {
+            statistics = new  QueryStatistics();
+            return pageNumber == 1
+                ? queryable.As<IMartenQueryable<T>>().Take(pageSize)
+                : queryable.As<IMartenQueryable<T>>().Skip((pageNumber - 1) * pageSize)
+                    .Take(pageSize);
+        }
+
         return pageNumber == 1
             ? queryable.As<IMartenQueryable<T>>().Stats(out statistics).Take(pageSize)
             : queryable.As<IMartenQueryable<T>>().Stats(out statistics).Skip((pageNumber - 1) * pageSize)

src/Marten/Pagination/PagedListQueryableExtensions.cs

@@ -25,7 +25,44 @@ public static IPagedList<T> ToPagedList<T>(
         int pageSize)
     {
         // return paged list
-        return PagedList<T>.Create(queryable, pageNumber, pageSize);
+        return PagedList<T>.Create(queryable, pageNumber, pageSize, false);
+    }
+
+    /// <summary>
+    ///     Extension method to return a paged results using separate count query
+    /// </summary>
+    /// <typeparam name="T">Document Type</typeparam>
+    /// <param name="queryable">Extension point on <see cref="IQueryable{T}" /></param>
+    /// <param name="pageNumber">one based page number</param>
+    /// <param name="pageSize">Page size</param>
+    /// <param name="useCountQuery">Use a separate count query rather than using Stats.</param>
+    /// <returns>return paged result</returns>
+    public static IPagedList<T> ToPagedList<T>(
+        this IQueryable<T> queryable,
+        int pageNumber,
+        int pageSize, bool useQueryCount)
+    {
+        // return paged list
+        return PagedList<T>.Create(queryable, pageNumber, pageSize, useQueryCount);
+    }
+
+    /// <summary>
+    ///     Async Extension method to return a paged results
+    /// </summary>
+    /// <typeparam name="T">Document Type</typeparam>
+    /// <param name="queryable">Extension point on <see cref="IQueryable{T}" /></param>
+    /// <param name="pageNumber">One based page number</param>
+    /// <param name="pageSize">Page size</param>
+    /// <param name="token">Cancellation token</param>
+    /// <returns>return paged result</returns>
+    public static async Task<IPagedList<T>> ToPagedListAsync<T>(
+        this IQueryable<T> queryable,
+        int pageNumber,
+        int pageSize,
+        CancellationToken token = default)
+    {
+        // return paged list
+        return await PagedList<T>.CreateAsync(queryable, pageNumber, pageSize, false, token).ConfigureAwait(false);
     }
 
     /// <summary>
@@ -36,14 +73,16 @@ public static IPagedList<T> ToPagedList<T>(
     /// <param name="pageNumber">One based page number</param>
     /// <param name="pageSize">Page size</param>
     /// <param name="token">Cancellation token</param>
+    /// <param name="useCountQuery">Use a separate count query rather than using Stats.</param>
     /// <returns>return paged result</returns>
     public static async Task<IPagedList<T>> ToPagedListAsync<T>(
         this IQueryable<T> queryable,
         int pageNumber,
         int pageSize,
+        bool useCountQuery,
         CancellationToken token = default)
     {
         // return paged list
-        return await PagedList<T>.CreateAsync(queryable, pageNumber, pageSize, token).ConfigureAwait(false);
+        return await PagedList<T>.CreateAsync(queryable, pageNumber, pageSize, useCountQuery, token).ConfigureAwait(false);
     }
 }