Concurrent file listing (#4401)

* show how many files are added when loading remake of #4388 * add french (cherry pick) * implement concurrent file listing
transloadit · Apr 15, 2023 · d024eb0 · d024eb0
1 parent 465dec7
commit d024eb0
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 25 deletions.
diff --git a/packages/@uppy/provider-views/package.json b/packages/@uppy/provider-views/package.json
@@ -23,6 +23,7 @@
     "@uppy/utils": "workspace:^",
     "classnames": "^2.2.6",
     "nanoid": "^4.0.0",
+    "p-queue": "^7.3.4",
     "preact": "^10.5.13"
   },
   "peerDependencies": {

diff --git a/packages/@uppy/provider-views/src/ProviderView/ProviderView.jsx b/packages/@uppy/provider-views/src/ProviderView/ProviderView.jsx
@@ -1,4 +1,6 @@
 import { h } from 'preact'
+// eslint-disable-next-line import/no-unresolved
+import PQueue from 'p-queue'
 
 import { getSafeFileId } from '@uppy/utils/lib/generateFileID'
 
@@ -235,23 +237,23 @@ export default class ProviderView extends View {
     }
   }
 
-  async* recursivelyListAllFiles (path) {
+  async recursivelyListAllFiles (path, queue, onFiles) {
     let curPath = path
 
-    // need to repeat the list call until there are no more pages
     while (curPath) {
       const res = await this.provider.list(curPath)
+      curPath = res.nextPagePath
 
-      for (const item of res.items) {
-        if (item.isFolder) {
-          // recursively call self for folder
-          yield* this.recursivelyListAllFiles(item.requestPath)
-        } else {
-          yield item
-        }
-      }
+      const files = res.items.filter((item) => !item.isFolder)
+      const folders = res.items.filter((item) => item.isFolder)
 
-      curPath = res.nextPagePath
+      onFiles(files)
+
+      // recursively queue call to self for each folder
+      const promises = folders.map(async (folder) => queue.add(async () => (
+        this.recursivelyListAllFiles(folder.requestPath, queue, onFiles)
+      )))
+      await Promise.all(promises) // in case we get an error
     }
   }
 
@@ -263,28 +265,34 @@ export default class ProviderView extends View {
       const messages = []
       const newFiles = []
 
-      // eslint-disable-next-line no-unreachable-loop
       for (const file of currentSelection) {
         if (file.isFolder) {
           const { requestPath, name } = file
           let isEmpty = true
           let numNewFiles = 0
 
-          for await (const fileInFolder of this.recursivelyListAllFiles(requestPath)) {
-            const tagFile = this.getTagFile(fileInFolder)
-            const id = getSafeFileId(tagFile)
-            // If the same folder is added again, we don't want to send
-            // X amount of duplicate file notifications, we want to say
-            // the folder was already added. This checks if all files are duplicate,
-            // if that's the case, we don't add the files.
-            if (!this.plugin.uppy.checkIfFileAlreadyExists(id)) {
-              newFiles.push(fileInFolder)
-              numNewFiles++
-              this.setLoading(this.plugin.uppy.i18n('addedNumFiles', { numFiles: numNewFiles }))
+          const queue = new PQueue({ concurrency: 6 })
+
+          const onFiles = (files) => {
+            for (const newFile of files) {
+              const tagFile = this.getTagFile(newFile)
+              const id = getSafeFileId(tagFile)
+              // If the same folder is added again, we don't want to send
+              // X amount of duplicate file notifications, we want to say
+              // the folder was already added. This checks if all files are duplicate,
+              // if that's the case, we don't add the files.
+              if (!this.plugin.uppy.checkIfFileAlreadyExists(id)) {
+                newFiles.push(newFile)
+                numNewFiles++
+                this.setLoading(this.plugin.uppy.i18n('addedNumFiles', { numFiles: numNewFiles }))
+              }
+              isEmpty = false
             }
-            isEmpty = false
           }
 
+          await this.recursivelyListAllFiles(requestPath, queue, onFiles)
+          await queue.onIdle()
+
           let message
           if (isEmpty) {
             message = this.plugin.uppy.i18n('emptyFolderAdded')

diff --git a/yarn.lock b/yarn.lock
@@ -8952,6 +8952,7 @@ __metadata:
     "@uppy/utils": "workspace:^"
     classnames: ^2.2.6
     nanoid: ^4.0.0
+    p-queue: ^7.3.4
     preact: ^10.5.13
   peerDependencies:
     "@uppy/core": "workspace:^"
@@ -16688,7 +16689,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"eventemitter3@npm:^4.0.0":
+"eventemitter3@npm:^4.0.0, eventemitter3@npm:^4.0.7":
   version: 4.0.7
   resolution: "eventemitter3@npm:4.0.7"
   checksum: 1875311c42fcfe9c707b2712c32664a245629b42bb0a5a84439762dd0fd637fc54d078155ea83c2af9e0323c9ac13687e03cfba79b03af9f40c89b4960099374
@@ -25996,6 +25997,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"p-queue@npm:^7.3.4":
+  version: 7.3.4
+  resolution: "p-queue@npm:7.3.4"
+  dependencies:
+    eventemitter3: ^4.0.7
+    p-timeout: ^5.0.2
+  checksum: a21b8a4dd75f64a4988e4468cc344d1b45132506ddd2c771932d3de446d108ee68713b629e0d3f0809c227bc10eafc613edde6ae741d9f60db89b6031e40921c
+  languageName: node
+  linkType: hard
+
 "p-retry@npm:^4.5.0":
   version: 4.6.2
   resolution: "p-retry@npm:4.6.2"
@@ -26015,6 +26026,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"p-timeout@npm:^5.0.2":
+  version: 5.1.0
+  resolution: "p-timeout@npm:5.1.0"
+  checksum: f5cd4e17301ff1ff1d8dbf2817df0ad88c6bba99349fc24d8d181827176ad4f8aca649190b8a5b1a428dfd6ddc091af4606835d3e0cb0656e04045da5c9e270c
+  languageName: node
+  linkType: hard
+
 "p-try@npm:^1.0.0":
   version: 1.0.0
   resolution: "p-try@npm:1.0.0"