Merge pull request #183 from laravel/feat/improve-serve-assets

Removes 500 public assets limitation and makes files uploads concurrent
laravel · May 13, 2022 · fc201b3 · fc201b3
2 parents 9e93d9e + 5340a42
commit fc201b3
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 35 deletions.
diff --git a/src/Aws/AwsStorageProvider.php b/src/Aws/AwsStorageProvider.php
@@ -5,8 +5,9 @@
 use GuzzleHttp\Client;
 use GuzzleHttp\Pool;
 use GuzzleHttp\Psr7\Request;
+use Illuminate\Support\LazyCollection;
 use Laravel\VaporCli\ConsoleVaporClient;
-use Laravel\VaporCli\Exceptions\CopyRequestFailedException;
+use Laravel\VaporCli\Exceptions\RequestFailedException;
 use Laravel\VaporCli\Helpers;
 use Symfony\Component\Console\Helper\ProgressBar;
 
@@ -84,16 +85,68 @@ public function request($method, $url, array $headers = [])
         ]));
     }
 
+    /**
+     * Execute the given store requests.
+     *
+     * @param  array  $requests
+     * @param  string  $assetPath
+     * @param  callable  $callback
+     * @return void
+     */
+    public function executeStoreRequests($requests, $assetPath, $callback)
+    {
+        collect($requests)->chunk(10)->each(function ($chunkOfRequests) use ($assetPath, $callback) {
+            $requests = LazyCollection::make($chunkOfRequests)->map(function ($request) use ($assetPath) {
+                $file = $assetPath.'/'.$request['path'];
+                $request['stream'] = fopen($file, 'r+');
+
+                return $request;
+            });
+
+            $generator = function () use ($requests, $callback) {
+                foreach ($requests as $request) {
+                    $callback($request);
+
+                    yield new Request(
+                        'PUT',
+                        $request['url'],
+                        array_merge(
+                            $request['headers'],
+                            ['Cache-Control' => 'public, max-age=31536000']
+                        ),
+                        $request['stream']
+                    );
+                }
+            };
+
+            (new Pool(new Client(), $generator(), [
+                'concurrency' => 10,
+                'rejected' => function ($reason, $index) {
+                    throw new RequestFailedException($reason->getMessage(), $index);
+                },
+            ]))
+                ->promise()
+                ->wait();
+
+            $requests->each(function ($request) {
+                fclose($request['stream']);
+            });
+        });
+    }
+
     /**
      * Execute the given copy requests.
      *
      * @param  array  $requests
+     * @param  callable  $callback
      * @return void
      */
-    public function executeCopyRequests($requests)
+    public function executeCopyRequests($requests, $callback)
     {
-        $requests = function () use ($requests) {
+        $generator = function () use ($requests, $callback) {
             foreach ($requests as $request) {
+                $callback($request);
+
                 yield new Request(
                     'PUT',
                     $request['url'],
@@ -105,10 +158,10 @@ public function executeCopyRequests($requests)
             }
         };
 
-        (new Pool(new Client(), $requests(), [
+        (new Pool(new Client(), $generator(), [
             'concurrency' => 10,
             'rejected' => function ($reason, $index) {
-                throw new CopyRequestFailedException($reason->getMessage(), $index);
+                throw new RequestFailedException($reason->getMessage(), $index);
             },
         ]))
             ->promise()

diff --git a/...Exceptions/CopyRequestFailedException.php → src/Exceptions/RequestFailedException.php b/...Exceptions/CopyRequestFailedException.php → src/Exceptions/RequestFailedException.php
@@ -4,7 +4,7 @@
 
 use RuntimeException;
 
-class CopyRequestFailedException extends RuntimeException
+class RequestFailedException extends RuntimeException
 {
     /**
      * The file index.

diff --git a/src/ServeAssets.php b/src/ServeAssets.php
@@ -3,7 +3,7 @@
 namespace Laravel\VaporCli;
 
 use Laravel\VaporCli\Aws\AwsStorageProvider;
-use Laravel\VaporCli\Exceptions\CopyRequestFailedException;
+use Laravel\VaporCli\Exceptions\RequestFailedException;
 
 class ServeAssets
 {
@@ -18,21 +18,28 @@ class ServeAssets
     public function __invoke(ConsoleVaporClient $vapor, array $artifact, $fresh)
     {
         $assetPath = Path::build().'/assets';
-
-        $requests = $this->getAuthorizedAssetRequests(
-            $vapor,
-            $artifact,
-            $assetFiles = $this->getAssetFiles($assetPath),
-            $fresh
-        );
-
-        $this->executeStoreAssetRequests($requests['store'], $assetPath);
-        $this->executeCopyAssetRequests($requests['copy'], $assetPath);
-
-        $vapor->recordArtifactAssets(
-            $artifact['id'],
-            $assetFiles
-        );
+        $assetFiles = $this->getAssetFiles($assetPath);
+
+        collect($assetFiles)
+            ->chunk(400)
+            ->map
+            ->all()
+            ->each(function ($chunkOfAssetFiles) use ($assetPath, $vapor, $artifact, $fresh) {
+                $requests = $this->getAuthorizedAssetRequests(
+                    $vapor,
+                    $artifact,
+                    $chunkOfAssetFiles,
+                    $fresh
+                );
+
+                $this->executeStoreAssetRequests($requests['store'], $assetPath);
+                $this->executeCopyAssetRequests($requests['copy'], $assetPath);
+
+                $vapor->recordArtifactAssets(
+                    $artifact['id'],
+                    $chunkOfAssetFiles
+                );
+            });
     }
 
     /**
@@ -46,14 +53,19 @@ protected function executeStoreAssetRequests($requests, $assetPath)
     {
         $storage = Helpers::app(AwsStorageProvider::class);
 
-        foreach ($requests as $request) {
-            Helpers::step('<comment>Uploading Asset:</comment> '.$request['path'].' ('.Helpers::kilobytes($assetPath.'/'.$request['path']).')');
+        if (! empty($requests)) {
+            try {
+                $storage->executeStoreRequests($requests, $assetPath, function ($request) use ($assetPath) {
+                    Helpers::step('<comment>Uploading Asset:</comment> '.$request['path'].' ('.Helpers::kilobytes($assetPath.'/'.$request['path']).')');
+                });
+            } catch (RequestFailedException $e) {
+                $request = $requests[$e->getIndex()];
+
+                Helpers::line("<fg=red>Uploading:</> {$request['path']}");
+                Helpers::write($e->getMessage());
 
-            $storage->store(
-                $request['url'],
-                array_merge($request['headers'], ['Cache-Control' => 'public, max-age=31536000']),
-                $assetPath.'/'.$request['path']
-            );
+                exit(1);
+            }
         }
     }
 
@@ -68,14 +80,12 @@ protected function executeCopyAssetRequests($requests, $assetPath)
     {
         $storage = Helpers::app(AwsStorageProvider::class);
 
-        foreach ($requests as $request) {
-            Helpers::step('<fg=magenta>Copying Unchanged Asset:</> '.$request['path'].' ('.Helpers::kilobytes($assetPath.'/'.$request['path']).')');
-        }
-
         if (! empty($requests)) {
             try {
-                $storage->executeCopyRequests($requests);
-            } catch (CopyRequestFailedException $e) {
+                $storage->executeCopyRequests($requests, function ($request) use ($assetPath) {
+                    Helpers::step('<fg=magenta>Copying Unchanged Asset:</> '.$request['path'].' ('.Helpers::kilobytes($assetPath.'/'.$request['path']).')');
+                });
+            } catch (RequestFailedException $e) {
                 $request = $requests[$e->getIndex()];
 
                 Helpers::line("<fg=red>Copying:</> {$request['path']}");