Merge #166

166: Add methods to automatically add/update documents in batches r=curquiza a=alejandrocq

Hi,

I have added the methods to add/update documents in batches (#157) in `Index.cs`. I have not been able to run the tests correctly because I get the following error in most of them:

```
Meilisearch.MeilisearchApiError
MeilisearchApiError, Message: The Content-Type "application/json; charset=utf-8" is invalid. Accepted values for the Content-Type header are: "application/json", "application/x-ndjson", "application/csv"
```

Seems to be an error with the `PostJsonAsync`method adding the charset parameter, but I suppose that has been working correctly for you.

Let me know if the code is okay and what should I do to fix that error. I will keep investigating if I get some time.

Co-authored-by: Alejandro Castilla Quesada <[email protected]>

Author: bors[bot]

src/Meilisearch/Index.cs

@@ -133,6 +133,25 @@ public async Task<UpdateStatus> AddDocuments<T>(IEnumerable<T> documents, string
             return await responseMessage.Content.ReadFromJsonAsync<UpdateStatus>();
         }
 
+        /// <summary>
+        /// Adds documents in batches with size specified with <paramref name="batchSize"/>.
+        /// </summary>
+        /// <param name="documents">Documents to add.</param>
+        /// <param name="batchSize">Size of documents batches while adding them.</param>
+        /// <param name="primaryKey">Primary key for the documents.</param>
+        /// <typeparam name="T">Type of the document. Even though documents are schemaless in MeiliSearch, making it typed helps in compile time.</typeparam>
+        /// <returns>Returns the updateID of this async operation.</returns>
+        public async Task<IEnumerable<UpdateStatus>> AddDocumentsInBatches<T>(IEnumerable<T> documents, int batchSize = 1000, string primaryKey = default)
+        {
+            async Task AddAction(List<T> items, List<UpdateStatus> updates)
+            {
+                updates.Add(await this.AddDocuments(items, primaryKey));
+            }
+
+            var result = await BatchOperation(documents, batchSize, AddAction);
+            return result;
+        }
+
         /// <summary>
         /// Update documents.
         /// </summary>
@@ -155,6 +174,25 @@ public async Task<UpdateStatus> UpdateDocuments<T>(IEnumerable<T> documents, str
             return await responseMessage.Content.ReadFromJsonAsync<UpdateStatus>();
         }
 
+        /// <summary>
+        /// Updates documents in batches with size specified with <paramref name="batchSize"/>.
+        /// </summary>
+        /// <param name="documents">Documents to update.</param>
+        /// <param name="batchSize">Size of documents batches while updating them.</param>
+        /// <param name="primaryKey">Primary key for the documents.</param>
+        /// <typeparam name="T">Type of the document. Even though documents are schemaless in MeiliSearch, making it typed helps in compile time.</typeparam>
+        /// <returns>Returns the updateID of this async operation.</returns>
+        public async Task<IEnumerable<UpdateStatus>> UpdateDocumentsInBatches<T>(IEnumerable<T> documents, int batchSize = 1000, string primaryKey = default)
+        {
+            async Task UpdateAction(List<T> items, List<UpdateStatus> updates)
+            {
+                updates.Add(await this.UpdateDocuments(items, primaryKey));
+            }
+
+            var result = await BatchOperation(documents, batchSize, UpdateAction);
+            return result;
+        }
+
         /// <summary>
         /// Get document by its ID.
         /// </summary>
@@ -621,5 +659,19 @@ internal Index WithHttpClient(HttpRequest http)
             this.http = http;
             return this;
         }
+
+        private static async Task<List<UpdateStatus>> BatchOperation<T>(IEnumerable<T> items, int batchSize, Func<List<T>, List<UpdateStatus>, Task> action)
+        {
+            var itemsList = new List<T>(items);
+            var numberOfBatches = Math.Ceiling((double)itemsList.Count / batchSize);
+            var result = new List<UpdateStatus>();
+            for (var i = 0; i < numberOfBatches; i++)
+            {
+                var batch = itemsList.GetRange(i * batchSize, batchSize);
+                await action.Invoke(batch, result);
+            }
+
+            return result;
+        }
     }
 }

tests/Meilisearch.Tests/DocumentTests.cs

@@ -36,6 +36,35 @@ public async Task BasicDocumentsAddition()
             docs.First().Genre.Should().BeNull();
         }
 
+        [Fact]
+        public async Task BasicDocumentsAdditionInBatches()
+        {
+            var indexUID = "BasicDocumentsAdditionInBatchesTest";
+            Index index = this.client.Index(indexUID);
+
+            // Add the documents
+            Movie[] movies =
+            {
+                new Movie { Id = "1", Name = "Batman" },
+                new Movie { Id = "2", Name = "Reservoir Dogs" },
+                new Movie { Id = "3", Name = "Taxi Driver" },
+                new Movie { Id = "4", Name = "Interstellar" },
+            };
+            var updates = await index.AddDocumentsInBatches(movies, 2);
+            foreach (var u in updates)
+            {
+                u.UpdateId.Should().BeGreaterOrEqualTo(0);
+                await index.WaitForPendingUpdate(u.UpdateId);
+            }
+
+            // Check the documents have been added (one movie from each batch)
+            var docs = (await index.GetDocuments<Movie>()).ToList();
+            Assert.Equal("1", docs.ElementAt(0).Id);
+            Assert.Equal("Batman", docs.ElementAt(0).Name);
+            Assert.Equal("3", docs.ElementAt(2).Id);
+            Assert.Equal("Taxi Driver", docs.ElementAt(2).Name);
+        }
+
         [Fact]
         public async Task BasicDocumentsAdditionWithCreateIndex()
         {
@@ -124,6 +153,50 @@ public async Task BasicDocumentsUpdate()
             docs.ElementAt(1).Genre.Should().BeNull();
         }
 
+        [Fact]
+        public async Task BasicDocumentsUpdateInBatches()
+        {
+            var indexUID = "BasicDocumentsUpdateInBatchesTest";
+            Index index = this.client.Index(indexUID);
+
+            // Add the documents
+            Movie[] movies =
+            {
+                new Movie { Id = "1", Name = "Batman" },
+                new Movie { Id = "2", Name = "Reservoir Dogs" },
+                new Movie { Id = "3", Name = "Taxi Driver" },
+                new Movie { Id = "4", Name = "Interstellar" },
+            };
+            var updates = await index.AddDocumentsInBatches(movies, 2);
+            foreach (var u in updates)
+            {
+                u.UpdateId.Should().BeGreaterOrEqualTo(0);
+                await index.WaitForPendingUpdate(u.UpdateId);
+            }
+
+            movies = new Movie[]
+            {
+                new Movie { Id = "1", Name = "Batman", Genre = "Action" },
+                new Movie { Id = "2", Name = "Reservoir Dogs", Genre = "Drama" },
+                new Movie { Id = "3", Name = "Taxi Driver", Genre = "Drama" },
+                new Movie { Id = "4", Name = "Interstellar", Genre = "Sci-Fi" },
+            };
+            updates = await index.UpdateDocumentsInBatches(movies, 2);
+            foreach (var u in updates)
+            {
+                u.UpdateId.Should().BeGreaterOrEqualTo(0);
+                await index.WaitForPendingUpdate(u.UpdateId);
+            }
+
+            // Assert movies have genre after update
+            var docs = (await index.GetDocuments<Movie>()).ToList();
+            foreach (var movie in docs)
+            {
+                movie.Genre.Should().NotBeNull();
+                movie.Genre.Should().NotBeEmpty();
+            }
+        }
+
         [Fact]
         public async Task DocumentsUpdateWithPrimaryKey()
         {

tests/Meilisearch.Tests/SearchTests.cs

@@ -158,7 +158,7 @@ public async Task CustomSearchWithFilterWithSpaces()
                 });
             movies.Hits.Should().NotBeEmpty();
             movies.FacetsDistribution.Should().BeNull();
-            Assert.Equal(1, movies.Hits.Count());
+            Assert.Single(movies.Hits);
             Assert.Equal("1344", movies.Hits.First().Id);
             Assert.Equal("The Hobbit", movies.Hits.First().Name);
         }