From 8ac2e53365d462da501993bb9c82f3c06dd211df Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 10:28:51 -0700 Subject: [PATCH 1/9] feat: use UNION ALL queries in applySearch --- app/Services/Plugins/QueryPluginsService.php | 26 ++++++++++++++------ app/Utils/Regex.php | 5 ++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 742b320f..5af5cb2e 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -3,6 +3,7 @@ namespace App\Services\Plugins; use App\Models\WpOrg\Plugin; +use App\Utils\Regex; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; @@ -51,13 +52,24 @@ public function queryPlugins( /** @param Builder $query */ private static function applySearch(Builder $query, string $search): void { - $query->where(function (Builder $q) use ($search) { - $q - ->whereFullText('slug', $search) - ->orWhereFullText('name', $search) - ->orWhereFullText('short_description', $search) - ->orWhereFullText('description', $search); - }); + $slug = Regex::replace('/[^a-z0-9-]+/i', '-', $search); + $query->where('slug', $slug); // need an initial condition or it retrieves everything + + $q = Plugin::query(); + + $slug_contains = $q->clone()->where('slug', 'like', "%$slug%"); + $name_exact = $q->clone()->where('name', $search); + $name_contains = $q->clone()->where('name', 'like', "%$search%"); + $short_description_contains = $q->clone()->where('short_description', 'like', "%$search%"); + $short_description_fulltext = $q->clone()->whereFullText('short_description', $search); + $description_fulltext = $q->clone()->whereFullText('description', $search); + + $query->unionAll($slug_contains); + $query->unionAll($name_exact); + $query->unionAll($name_contains); + $query->unionAll($short_description_contains); + $query->unionAll($short_description_fulltext); + $query->unionAll($description_fulltext); } /** @param Builder $query */ diff --git a/app/Utils/Regex.php b/app/Utils/Regex.php index 396dac6b..65f7ce00 100644 --- a/app/Utils/Regex.php +++ b/app/Utils/Regex.php @@ -12,6 +12,11 @@ public static function match(string $pattern, string $subject): array return $matches; } + public static function replace(string $pattern, string $replacement, string $subject, int $limit = -1): string + { + return \Safe\preg_replace($pattern, $replacement, $subject, $limit); + } + private function __construct() { // not instantiable From 4f03fe872552e45645a70e08411c7c95a4786f43 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 10:29:02 -0700 Subject: [PATCH 2/9] fix: substring search on author --- app/Services/Plugins/QueryPluginsService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 5af5cb2e..4e85b683 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -75,7 +75,7 @@ private static function applySearch(Builder $query, string $search): void /** @param Builder $query */ private static function applyAuthor(Builder $query, string $author): void { - $query->whereLike('author', $author); + $query->whereLike('author', "%{$author}%"); } /** @param Builder $query */ From 3ea88cdae36b4d9de6da603e2647d3def11f6493 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 11:17:51 -0700 Subject: [PATCH 3/9] feat: add and use trigram indexes --- app/Services/Plugins/QueryPluginsService.php | 26 +++++++++++++------ .../2025_02_21_174738_add_trigram_indexes.php | 24 +++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 database/migrations/2025_02_21_174738_add_trigram_indexes.php diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 4e85b683..a8c221ad 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -27,6 +27,8 @@ public function queryPlugins( ?string $author = null, string $browse = 'popular', ): array { + $search = self::normalizeSearchString($search); + $query = Plugin::query() ->when($browse, self::applyBrowse(...)) ->when($search, self::applySearch(...)) @@ -57,18 +59,16 @@ private static function applySearch(Builder $query, string $search): void $q = Plugin::query(); - $slug_contains = $q->clone()->where('slug', 'like', "%$slug%"); + $slug_similar = $q->clone()->whereRaw("slug %> '$search'"); $name_exact = $q->clone()->where('name', $search); - $name_contains = $q->clone()->where('name', 'like', "%$search%"); - $short_description_contains = $q->clone()->where('short_description', 'like', "%$search%"); - $short_description_fulltext = $q->clone()->whereFullText('short_description', $search); + $name_similar = $q->clone()->whereRaw("name %> '$search'"); + $short_description_similar = $q->clone()->whereRaw("short_description %> '$search'"); $description_fulltext = $q->clone()->whereFullText('description', $search); - $query->unionAll($slug_contains); $query->unionAll($name_exact); - $query->unionAll($name_contains); - $query->unionAll($short_description_contains); - $query->unionAll($short_description_fulltext); + $query->unionAll($slug_similar); + $query->unionAll($name_similar); + $query->unionAll($short_description_similar); $query->unionAll($description_fulltext); } @@ -99,4 +99,14 @@ private static function applyBrowse(Builder $query, string $browse): void default => $query->reorder('active_installs', 'desc'), }; } + + private static function normalizeSearchString(?string $search): ?string + { + if ($search === null) { + return null; + } + $search = trim($search); + $search = Regex::replace('/\s+/i', ' ', $search); + return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + } } diff --git a/database/migrations/2025_02_21_174738_add_trigram_indexes.php b/database/migrations/2025_02_21_174738_add_trigram_indexes.php new file mode 100644 index 00000000..63962441 --- /dev/null +++ b/database/migrations/2025_02_21_174738_add_trigram_indexes.php @@ -0,0 +1,24 @@ + Date: Fri, 21 Feb 2025 13:18:10 -0700 Subject: [PATCH 4/9] wip: add trigram indexes to more fields, themes, authors --- app/Services/Plugins/QueryPluginsService.php | 4 ++- .../2025_02_21_174738_add_trigram_indexes.php | 26 ++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index a8c221ad..36078da7 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -28,6 +28,8 @@ public function queryPlugins( string $browse = 'popular', ): array { $search = self::normalizeSearchString($search); + $tag = self::normalizeSearchString($tag); + $author = self::normalizeSearchString($author); $query = Plugin::query() ->when($browse, self::applyBrowse(...)) @@ -75,7 +77,7 @@ private static function applySearch(Builder $query, string $search): void /** @param Builder $query */ private static function applyAuthor(Builder $query, string $author): void { - $query->whereLike('author', "%{$author}%"); + $query->whereRaw("author %> '$author'"); } /** @param Builder $query */ diff --git a/database/migrations/2025_02_21_174738_add_trigram_indexes.php b/database/migrations/2025_02_21_174738_add_trigram_indexes.php index 63962441..be3677cc 100644 --- a/database/migrations/2025_02_21_174738_add_trigram_indexes.php +++ b/database/migrations/2025_02_21_174738_add_trigram_indexes.php @@ -8,17 +8,31 @@ public function up(): void { DB::raw('CREATE EXTENSION IF NOT EXISTS pg_trgm'); - DB::raw('CREATE INDEX plugins_slug_trgm ON plugins USING GIST (slug gist_trgm_ops(siglen=48))'); - DB::raw('CREATE INDEX plugins_name_trgm ON plugins USING GIST (name gist_trgm_ops(siglen=48))'); + DB::raw('CREATE INDEX plugins_slug_trgm ON plugins USING GIST (slug gist_trgm_ops(siglen=32))'); + DB::raw('CREATE INDEX plugins_name_trgm ON plugins USING GIST (name gist_trgm_ops(siglen=32))'); DB::raw( - 'CREATE INDEX plugins_short_description_trgm ON plugins USING GIST (short_description gist_trgm_ops(siglen=48))', + 'CREATE INDEX plugins_short_description_trgm ON plugins USING GIST (short_description gist_trgm_ops(siglen=32))', ); + DB::raw('CREATE INDEX plugins_author_trgm ON plugins USING GIST (author gist_trgm_ops(siglen=32))'); + + DB::raw('CREATE INDEX themes_slug_trgm ON themes USING GIST (slug gist_trgm_ops(siglen=32))'); + DB::raw('CREATE INDEX themes_name_trgm ON themes USING GIST (name gist_trgm_ops(siglen=32))'); + + DB::raw( + 'CREATE INDEX author_user_nicename_trgm on authors using GIST (user_nicename gist_trgm_ops(siglen=32))', + ); + DB::raw('CREATE INDEX author_display_name_trgm on authors using GIST (display_name gist_trgm_ops(siglen=32))'); } public function down(): void { - DB::raw('DROP INDEX plugins_short_description_trgm'); - DB::raw('DROP INDEX plugins_name_trgm'); - DB::raw('DROP INDEX plugins_slug_trgm'); + DB::raw('DROP INDEX IF EXISTS author_display_name_trgm'); + DB::raw('DROP INDEX IF EXISTS author_user_nicename_trgm'); + DB::raw('DROP INDEX IF EXISTS themes_name_trgm'); + DB::raw('DROP INDEX IF EXISTS themes_slug_trgm'); + DB::raw('DROP INDEX IF EXISTS plugins_author_trgm'); + DB::raw('DROP INDEX IF EXISTS plugins_short_description_trgm'); + DB::raw('DROP INDEX IF EXISTS plugins_name_trgm'); + DB::raw('DROP INDEX IF EXISTS plugins_slug_trgm'); } }; From f17a1bbef2a20c4008ae63ea632a625e4fea0635 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 13:39:54 -0700 Subject: [PATCH 5/9] feat: make use of trigram search on themes --- app/Services/Themes/QueryThemesService.php | 43 ++++++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/app/Services/Themes/QueryThemesService.php b/app/Services/Themes/QueryThemesService.php index 4f5a3b75..0c06c6f7 100644 --- a/app/Services/Themes/QueryThemesService.php +++ b/app/Services/Themes/QueryThemesService.php @@ -6,6 +6,7 @@ use App\Http\Resources\ThemeCollection; use App\Http\Resources\ThemeResource; use App\Models\WpOrg\Theme; +use App\Utils\Regex; use Illuminate\Database\Eloquent\Builder; class QueryThemesService @@ -16,12 +17,15 @@ public function queryThemes(QueryThemesRequest $req): ThemeCollection $perPage = $req->per_page; $skip = ($page - 1) * $perPage; + $search = self::normalizeSearchString($req->search); + $author = self::normalizeSearchString($req->author); + $themesBaseQuery = Theme::query() ->orderBy('last_updated', 'desc') // default sort ->when($req->browse, self::applyBrowse(...)) - ->when($req->search, self::applySearch(...)) + ->when($search, self::applySearch(...)) ->when($req->theme, self::applyTheme(...)) - ->when($req->author, self::applyAuthor(...)) + ->when($author, self::applyAuthor(...)) ->when($req->tags, self::applyTags(...)); $total = $themesBaseQuery->count(); @@ -52,22 +56,35 @@ private static function applyBrowse(Builder $query, string $browse): void /** @param Builder $query */ private static function applySearch(Builder $query, string $search): void { - $query - ->whereFullText('slug', $search) - ->orWhereFullText('name', $search) - ->orWhereFullText('description', $search); + $slug = Regex::replace('/[^a-z0-9-]+/i', '-', $search); + $query->where('slug', $slug); // need an initial condition or it retrieves everything + + $q = Theme::query(); + + $slug_similar = $q->clone()->whereRaw("slug %> '$search'"); + $name_exact = $q->clone()->where('name', $search); + $name_similar = $q->clone()->whereRaw("name %> '$search'"); + $description_fulltext = $q->clone()->whereFullText('description', $search); + + $query->unionAll($name_exact); + $query->unionAll($slug_similar); + $query->unionAll($name_similar); + $query->unionAll($description_fulltext); } /** @param Builder $query */ private static function applyTheme(Builder $query, string $theme): void { - $query->whereFullText('slug', $theme); + $query->whereRaw("slug %> '$theme'"); } /** @param Builder $query */ private static function applyAuthor(Builder $query, string $author): void { - $query->whereHas('author', fn(Builder $q) => $q->where('user_nicename', 'like', "%$author%")); + $query->whereHas( + 'author', + fn(Builder $q) => $q->whereRaw("user_nicename %> '$author'")->orWhereRaw("display_name %> '$author'"), + ); } /** @@ -78,4 +95,14 @@ private static function applyTags(Builder $query, array $tags): void { $query->whereHas('tags', fn(Builder $q) => $q->whereIn('slug', $tags)); } + + private static function normalizeSearchString(?string $search): ?string + { + if ($search === null) { + return null; + } + $search = trim($search); + $search = Regex::replace('/\s+/i', ' ', $search); + return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + } } From c87c551076856579d3c06b358829033344086b39 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 13:47:32 -0700 Subject: [PATCH 6/9] fix: uniq results by slug so at least each page has unique results --- app/Services/Plugins/QueryPluginsService.php | 3 ++- app/Services/Themes/QueryThemesService.php | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 36078da7..91086c79 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -43,7 +43,8 @@ public function queryPlugins( $plugins = $query ->offset(($page - 1) * $perPage) ->limit($perPage) - ->get(); + ->get() + ->unique('slug'); return [ 'plugins' => $plugins, diff --git a/app/Services/Themes/QueryThemesService.php b/app/Services/Themes/QueryThemesService.php index 0c06c6f7..36f325be 100644 --- a/app/Services/Themes/QueryThemesService.php +++ b/app/Services/Themes/QueryThemesService.php @@ -37,6 +37,7 @@ public function queryThemes(QueryThemesRequest $req): ThemeCollection ->get(); $collection = collect($themes) + ->unique('slug') ->map(fn($theme) => (new ThemeResource($theme))->additional(['fields' => $req->fields])); return new ThemeCollection($collection, $page, (int) ceil($total / $perPage), $total); From 453a21d28e807f3e6b17d8c36a2ddfcbbf7416c9 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 14:03:39 -0700 Subject: [PATCH 7/9] wip: add comment about not using placeholders --- app/Services/Plugins/QueryPluginsService.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 91086c79..5837e59a 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -62,6 +62,9 @@ private static function applySearch(Builder $query, string $search): void $q = Plugin::query(); + // I can't make %> work this way, only whereRaw works. TODO: find out why. + // $slug_similar = $q->clone()->where('slug', '%>', $search); + $slug_similar = $q->clone()->whereRaw("slug %> '$search'"); $name_exact = $q->clone()->where('name', $search); $name_similar = $q->clone()->whereRaw("name %> '$search'"); From 7f79207a21ffe76be220a5b1f436d26d63cdc870 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 14:12:09 -0700 Subject: [PATCH 8/9] fix: collapse consecutive dashes --- app/Services/Plugins/QueryPluginsService.php | 3 ++- app/Services/Themes/QueryThemesService.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 5837e59a..226b5e8a 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -113,6 +113,7 @@ private static function normalizeSearchString(?string $search): ?string } $search = trim($search); $search = Regex::replace('/\s+/i', ' ', $search); - return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + $search = Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + return Regex::replace('/-+/', '-', $search); // collapse any consecutive dashes from the above } } diff --git a/app/Services/Themes/QueryThemesService.php b/app/Services/Themes/QueryThemesService.php index 36f325be..1ad30920 100644 --- a/app/Services/Themes/QueryThemesService.php +++ b/app/Services/Themes/QueryThemesService.php @@ -104,6 +104,7 @@ private static function normalizeSearchString(?string $search): ?string } $search = trim($search); $search = Regex::replace('/\s+/i', ' ', $search); - return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + $search = Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset + return Regex::replace('/-+/', '-', $search); // collapse any consecutive dashes from the above } } From d8faa4124472dae250a548c996464b8520197937 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 21 Feb 2025 14:19:52 -0700 Subject: [PATCH 9/9] Revert "fix: collapse consecutive dashes" This reverts commit 7f79207a21ffe76be220a5b1f436d26d63cdc870. --- app/Services/Plugins/QueryPluginsService.php | 3 +-- app/Services/Themes/QueryThemesService.php | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/Services/Plugins/QueryPluginsService.php b/app/Services/Plugins/QueryPluginsService.php index 226b5e8a..5837e59a 100644 --- a/app/Services/Plugins/QueryPluginsService.php +++ b/app/Services/Plugins/QueryPluginsService.php @@ -113,7 +113,6 @@ private static function normalizeSearchString(?string $search): ?string } $search = trim($search); $search = Regex::replace('/\s+/i', ' ', $search); - $search = Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset - return Regex::replace('/-+/', '-', $search); // collapse any consecutive dashes from the above + return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset } } diff --git a/app/Services/Themes/QueryThemesService.php b/app/Services/Themes/QueryThemesService.php index 1ad30920..36f325be 100644 --- a/app/Services/Themes/QueryThemesService.php +++ b/app/Services/Themes/QueryThemesService.php @@ -104,7 +104,6 @@ private static function normalizeSearchString(?string $search): ?string } $search = trim($search); $search = Regex::replace('/\s+/i', ' ', $search); - $search = Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset - return Regex::replace('/-+/', '-', $search); // collapse any consecutive dashes from the above + return Regex::replace('/[^\w.,!?@#$_-]/i', ' ', $search); // strip most punctuation, allow a small subset } }