Smarter and faster plugin and themes searching

app/Services/Plugins/QueryPluginsService.php

@@ -3,6 +3,7 @@
 namespace App\Services\Plugins;
 
 use App\Models\WpOrg\Plugin;
+use App\Utils\Regex;
 use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Collection;
 
@@ -26,6 +27,10 @@ public function queryPlugins(
         ?string $author = null,
         string $browse = 'popular',
     ): array {
+        $search = self::normalizeSearchString($search);
+        $tag = self::normalizeSearchString($tag);
+        $author = self::normalizeSearchString($author);
+
         $query = Plugin::query()
             ->when($browse, self::applyBrowse(...))
             ->when($search, self::applySearch(...))
@@ -38,7 +43,8 @@ public function queryPlugins(
         $plugins = $query
             ->offset(($page - 1) * $perPage)
             ->limit($perPage)
-            ->get();
+            ->get()
+            ->unique('slug');
 
         return [
             'plugins' => $plugins,
@@ -51,19 +57,28 @@ public function queryPlugins(
     /** @param Builder<Plugin> $query */
     private static function applySearch(Builder $query, string $search): void
     {
-        $query->where(function (Builder $q) use ($search) {
-            $q
-                ->whereFullText('slug', $search)
-                ->orWhereFullText('name', $search)
-                ->orWhereFullText('short_description', $search)
-                ->orWhereFullText('description', $search);
-        });
+        $slug = Regex::replace('/[^a-z0-9-]+/i', '-', $search);
+        $query->where('slug', $slug); // need an initial condition or it retrieves everything
+
+        $q = Plugin::query();
+
+        $slug_similar = $q->clone()->whereRaw("slug %> '$search'");
+        $name_exact = $q->clone()->where('name', $search);
+        $name_similar = $q->clone()->whereRaw("name %> '$search'");
+        $short_description_similar = $q->clone()->whereRaw("short_description %> '$search'");
+        $description_fulltext = $q->clone()->whereFullText('description', $search);
+
+        $query->unionAll($name_exact);
+        $query->unionAll($slug_similar);
+        $query->unionAll($name_similar);
+        $query->unionAll($short_description_similar);
+        $query->unionAll($description_fulltext);
     }
 
     /** @param Builder<Plugin> $query */
     private static function applyAuthor(Builder $query, string $author): void
     {
-        $query->whereLike('author', $author);
+        $query->whereRaw("author %> '$author'");
     }
 
     /** @param Builder<Plugin> $query */
@@ -87,4 +102,14 @@ private static function applyBrowse(Builder $query, string $browse): void
             default => $query->reorder('active_installs', 'desc'),
         };
     }
+
+    private static function normalizeSearchString(?string $search): ?string
+    {
+        if ($search === null) {
+            return null;
+        }
+        $search = trim($search);
+        $search = Regex::replace('/\s+/i', ' ', $search);
+        return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset
+    }
 }

app/Services/Themes/QueryThemesService.php

@@ -6,6 +6,7 @@
 use App\Http\Resources\ThemeCollection;
 use App\Http\Resources\ThemeResource;
 use App\Models\WpOrg\Theme;
+use App\Utils\Regex;
 use Illuminate\Database\Eloquent\Builder;
 
 class QueryThemesService
@@ -16,12 +17,15 @@ public function queryThemes(QueryThemesRequest $req): ThemeCollection
         $perPage = $req->per_page;
         $skip = ($page - 1) * $perPage;
 
+        $search = self::normalizeSearchString($req->search);
+        $author = self::normalizeSearchString($req->author);
+
         $themesBaseQuery = Theme::query()
             ->orderBy('last_updated', 'desc')   // default sort
             ->when($req->browse, self::applyBrowse(...))
-            ->when($req->search, self::applySearch(...))
+            ->when($search, self::applySearch(...))
             ->when($req->theme, self::applyTheme(...))
-            ->when($req->author, self::applyAuthor(...))
+            ->when($author, self::applyAuthor(...))
             ->when($req->tags, self::applyTags(...));
 
         $total = $themesBaseQuery->count();
@@ -33,6 +37,7 @@ public function queryThemes(QueryThemesRequest $req): ThemeCollection
             ->get();
 
         $collection = collect($themes)
+            ->unique('slug')
             ->map(fn($theme) => (new ThemeResource($theme))->additional(['fields' => $req->fields]));
 
         return new ThemeCollection($collection, $page, (int) ceil($total / $perPage), $total);
@@ -52,22 +57,35 @@ private static function applyBrowse(Builder $query, string $browse): void
     /** @param Builder<Theme> $query */
     private static function applySearch(Builder $query, string $search): void
     {
-        $query
-            ->whereFullText('slug', $search)
-            ->orWhereFullText('name', $search)
-            ->orWhereFullText('description', $search);
+        $slug = Regex::replace('/[^a-z0-9-]+/i', '-', $search);
+        $query->where('slug', $slug); // need an initial condition or it retrieves everything
+
+        $q = Theme::query();
+
+        $slug_similar = $q->clone()->whereRaw("slug %> '$search'");
+        $name_exact = $q->clone()->where('name', $search);
+        $name_similar = $q->clone()->whereRaw("name %> '$search'");
+        $description_fulltext = $q->clone()->whereFullText('description', $search);
+
+        $query->unionAll($name_exact);
+        $query->unionAll($slug_similar);
+        $query->unionAll($name_similar);
+        $query->unionAll($description_fulltext);
     }
 
     /** @param Builder<Theme> $query */
     private static function applyTheme(Builder $query, string $theme): void
     {
-        $query->whereFullText('slug', $theme);
+        $query->whereRaw("slug %> '$theme'");
     }
 
     /** @param Builder<Theme> $query */
     private static function applyAuthor(Builder $query, string $author): void
     {
-        $query->whereHas('author', fn(Builder $q) => $q->where('user_nicename', 'like', "%$author%"));
+        $query->whereHas(
+            'author',
+            fn(Builder $q) => $q->whereRaw("user_nicename %> '$author'")->orWhereRaw("display_name %> '$author'"),
+        );
     }
 
     /**
@@ -78,4 +96,14 @@ private static function applyTags(Builder $query, array $tags): void
     {
         $query->whereHas('tags', fn(Builder $q) => $q->whereIn('slug', $tags));
     }
+
+    private static function normalizeSearchString(?string $search): ?string
+    {
+        if ($search === null) {
+            return null;
+        }
+        $search = trim($search);
+        $search = Regex::replace('/\s+/i', ' ', $search);
+        return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset
+    }
 }

app/Utils/Regex.php

@@ -12,6 +12,11 @@ public static function match(string $pattern, string $subject): array
         return $matches;
     }
 
+    public static function replace(string $pattern, string $replacement, string $subject, int $limit = -1): string
+    {
+        return \Safe\preg_replace($pattern, $replacement, $subject, $limit);
+    }
+
     private function __construct()
     {
         // not instantiable

database/migrations/2025_02_21_174738_add_trigram_indexes.php

@@ -0,0 +1,38 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    public function up(): void
+    {
+        DB::raw('CREATE EXTENSION IF NOT EXISTS pg_trgm');
+        DB::raw('CREATE INDEX plugins_slug_trgm ON plugins USING GIST (slug gist_trgm_ops(siglen=32))');
+        DB::raw('CREATE INDEX plugins_name_trgm ON plugins USING GIST (name gist_trgm_ops(siglen=32))');
+        DB::raw(
+            'CREATE INDEX plugins_short_description_trgm ON plugins USING GIST (short_description gist_trgm_ops(siglen=32))',
+        );
+        DB::raw('CREATE INDEX plugins_author_trgm ON plugins USING GIST (author gist_trgm_ops(siglen=32))');
+
+        DB::raw('CREATE INDEX themes_slug_trgm ON themes USING GIST (slug gist_trgm_ops(siglen=32))');
+        DB::raw('CREATE INDEX themes_name_trgm ON themes USING GIST (name gist_trgm_ops(siglen=32))');
+
+        DB::raw(
+            'CREATE INDEX author_user_nicename_trgm on authors using GIST (user_nicename gist_trgm_ops(siglen=32))',
+        );
+        DB::raw('CREATE INDEX author_display_name_trgm on authors using GIST (display_name gist_trgm_ops(siglen=32))');
+    }
+
+    public function down(): void
+    {
+        DB::raw('DROP INDEX IF EXISTS author_display_name_trgm');
+        DB::raw('DROP INDEX IF EXISTS author_user_nicename_trgm');
+        DB::raw('DROP INDEX IF EXISTS themes_name_trgm');
+        DB::raw('DROP INDEX IF EXISTS themes_slug_trgm');
+        DB::raw('DROP INDEX IF EXISTS plugins_author_trgm');
+        DB::raw('DROP INDEX IF EXISTS plugins_short_description_trgm');
+        DB::raw('DROP INDEX IF EXISTS plugins_name_trgm');
+        DB::raw('DROP INDEX IF EXISTS plugins_slug_trgm');
+    }
+};