Skip to content

Add multi-modal search #783

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Strift merged 12 commits into from
Sep 10, 2025
Merged

Add multi-modal search #783

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
24297f7
Scaffold test
Strift Sep 5, 2025
02d34d1
Add multi-modal search tests
Strift Sep 5, 2025
8587bab
Lint
Strift Sep 5, 2025
ffa5555
Skip tests when API key is missing
Strift Sep 5, 2025
0a5a178
Lint
Strift Sep 5, 2025
38637a2
Add code sample
Strift Sep 5, 2025
0dfa0e6
Skip test case if API key is not available
Strift Sep 5, 2025
d39da8e
Lint
Strift Sep 5, 2025
33d4023
Make private helper static, move to bottom
Strift Sep 10, 2025
7414150
Use file_get_contents
Strift Sep 10, 2025
d369703
Add JSON_THROW_ON_ERROR flag
Strift Sep 10, 2025
36452f5
Refactor test skipping
Strift Sep 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .code-samples.meilisearch.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,3 +808,18 @@ update_embedders_1: |-
'documentTemplate' => 'A document titled '{{doc.title}}' whose description starts with {{doc.overview|truncatewords: 20}}'
]
]);
search_parameter_reference_media_1: |-
$client->index('INDEX_NAME')->search('a futuristic movie', [
'hybrid' => [
'embedder' => 'EMBEDDER_NAME'
],
'media' => [
'textAndPoster' => [
'text' => 'a futuristic movie',
'image' => [
'mime' => 'image/jpeg',
'data' => 'base64EncodedImageData'
]
]
]
]);
196 changes: 196 additions & 0 deletions tests/Endpoints/MultiModalSearchTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
<?php

declare(strict_types=1);

namespace Tests\Endpoints;

use Meilisearch\Endpoints\Indexes;
use Meilisearch\Http\Client;
use Tests\TestCase;

final class MultiModalSearchTest extends TestCase
{
private Indexes $index;
private array $documents;

protected function setUp(): void
{
parent::setUp();

$http = new Client($this->host, getenv('MEILISEARCH_API_KEY'));
$http->patch('/experimental-features', ['multimodal' => true]);

$voyageApiKey = getenv('VOYAGE_API_KEY');
if (false === $voyageApiKey || '' === $voyageApiKey) {
self::markTestSkipped('Missing `VOYAGE_API_KEY` environment variable');
}

$this->index = $this->createEmptyIndex($this->safeIndexName());
$updateSettingsPromise = $this->index->updateSettings([
'searchableAttributes' => ['title', 'overview'],
'embedders' => [
'multimodal' => self::getVoyageEmbedderConfig($voyageApiKey),
],
]);
$this->index->waitForTask($updateSettingsPromise['taskUid']);

// Load the movies.json dataset
$documentsJson = file_get_contents('./tests/datasets/movies.json');
$this->documents = json_decode($documentsJson, true, 512, JSON_THROW_ON_ERROR);
$addDocumentsPromise = $this->index->addDocuments($this->documents);
$this->index->waitForTask($addDocumentsPromise['taskUid']);
}

public function testTextOnlySearch(): void
{
$query = 'A movie with lightsabers in space';
$response = $this->index->search($query, [
'media' => [
'text' => ['text' => $query],
],
'hybrid' => [
'embedder' => 'multimodal',
'semanticRatio' => 1,
],
]);
self::assertSame('Star Wars', $response->getHits()[0]['title']);
}

public function testImageOnlySearch(): void
{
$theFifthElementPoster = $this->documents[3]['poster'];
$response = $this->index->search(null, [
'media' => [
'poster' => [
'poster' => $theFifthElementPoster,
],
],
'hybrid' => [
'embedder' => 'multimodal',
'semanticRatio' => 1,
],
]);
self::assertSame('The Fifth Element', $response->getHits()[0]['title']);
}

public function testTextAndImageSearch(): void
{
$query = 'a futuristic movie';
$masterYodaBase64 = base64_encode(file_get_contents('./tests/assets/master-yoda.jpeg'));
$response = $this->index->search(null, [
'media' => [
'textAndPoster' => [
'text' => $query,
'image' => [
'mime' => 'image/jpeg',
'data' => $masterYodaBase64,
],
],
],
'hybrid' => [
'embedder' => 'multimodal',
'semanticRatio' => 1,
],
]);
self::assertSame('Star Wars', $response->getHits()[0]['title']);
}

private static function getVoyageEmbedderConfig(string $voyageApiKey): array
{
return [
'source' => 'rest',
'url' => 'https://api.voyageai.com/v1/multimodalembeddings',
'apiKey' => $voyageApiKey,
'dimensions' => 1024,
'indexingFragments' => [
'textAndPoster' => [
// the shape of the data here depends on the model used
'value' => [
'content' => [
[
'type' => 'text',
'text' => 'A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.',
],
[
'type' => 'image_url',
'image_url' => '{{doc.poster}}',
],
],
],
],
'text' => [
'value' => [
// The shape of the data here depends on the model used
'content' => [
[
'type' => 'text',
'text' => 'A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.',
],
],
],
],
'poster' => [
'value' => [
// The shape of the data here depends on the model used
'content' => [
[
'type' => 'image_url',
'image_url' => '{{doc.poster}}',
],
],
],
],
],
'searchFragments' => [
'textAndPoster' => [
'value' => [
'content' => [
[
'type' => 'text',
'text' => '{{media.textAndPoster.text}}',
],
[
'type' => 'image_base64',
'image_base64' => 'data:{{media.textAndPoster.image.mime}};base64,{{media.textAndPoster.image.data}}',
],
],
],
],
'text' => [
'value' => [
'content' => [
[
'type' => 'text',
'text' => '{{media.text.text}}',
],
],
],
],
'poster' => [
'value' => [
'content' => [
[
'type' => 'image_url',
'image_url' => '{{media.poster.poster}}',
],
],
],
],
],
'request' => [
// This request object matches the Voyage API request object
'inputs' => ['{{fragment}}', '{{..}}'],
'model' => 'voyage-multimodal-3',
],
'response' => [
// This response object matches the Voyage API response object
'data' => [
[
'embedding' => '{{embedding}}',
],
'{{..}}',
],
],
];
}
}
Binary file added tests/assets/master-yoda.jpeg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
133 changes: 133 additions & 0 deletions tests/datasets/movies.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
[
{
"id": 11,
"title": "Star Wars",
"overview": "Princess Leia is captured and held hostage by the evil Imperial forces in their effort to take over the galactic Empire. Venturesome Luke Skywalker and dashing captain Han Solo team together with the loveable robot duo R2-D2 and C-3PO to rescue the beautiful princess and restore peace and justice in the Empire.",
"genres": [
"Adventure",
"Action",
"Science Fiction"
],
"poster": "https://image.tmdb.org/t/p/w500/6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg",
"release_date": 233366400
},
{
"id": 12,
"title": "Finding Nemo",
"overview": "Nemo, an adventurous young clownfish, is unexpectedly taken from his Great Barrier Reef home to a dentist's office aquarium. It's up to his worrisome father Marlin and a friendly but forgetful fish Dory to bring Nemo home -- meeting vegetarian sharks, surfer dude turtles, hypnotic jellyfish, hungry seagulls, and more along the way.",
"genres": [
"Animation",
"Family"
],
"poster": "https://image.tmdb.org/t/p/w500/eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg",
"release_date": 1054252800
},
{
"id": 13,
"title": "Forrest Gump",
"overview": "A man with a low IQ has accomplished great things in his life and been present during significant historic events—in each case, far exceeding what anyone imagined he could do. But despite all he has achieved, his one true love eludes him.",
"genres": [
"Comedy",
"Drama",
"Romance"
],
"poster": "https://image.tmdb.org/t/p/w500/h5J4W4veyxMXDMjeNxZI46TsHOb.jpg",
"release_date": 773452800
},
{
"id": 18,
"title": "The Fifth Element",
"overview": "In 2257, a taxi driver is unintentionally given the task of saving a young girl who is part of the key that will ensure the survival of humanity.",
"genres": [
"Adventure",
"Fantasy",
"Action",
"Thriller",
"Science Fiction"
],
"poster": "https://image.tmdb.org/t/p/w500/fPtlCO1yQtnoLHOwKtWz7db6RGU.jpg",
"release_date": 862531200
},
{
"id": 22,
"title": "Pirates of the Caribbean: The Curse of the Black Pearl",
"overview": "Jack Sparrow, a freewheeling 18th-century pirate, quarrels with a rival pirate bent on pillaging Port Royal. When the governor's daughter is kidnapped, Sparrow decides to help the girl's love save her.",
"genres": [
"Adventure",
"Fantasy",
"Action"
],
"poster": "https://image.tmdb.org/t/p/w500/z8onk7LV9Mmw6zKz4hT6pzzvmvl.jpg",
"release_date": 1057708800
},
{
"id": 24,
"title": "Kill Bill: Vol. 1",
"overview": "An assassin is shot by her ruthless employer, Bill, and other members of their assassination circle – but she lives to plot her vengeance.",
"genres": [
"Action",
"Crime"
],
"poster": "https://image.tmdb.org/t/p/w500/v7TaX8kXMXs5yFFGR41guUDNcnB.jpg",
"release_date": 1065744000
},
{
"id": 35,
"title": "The Simpsons Movie",
"overview": "After Homer accidentally pollutes the town's water supply, Springfield is encased in a gigantic dome by the EPA and the Simpsons are declared fugitives.",
"genres": [
"Animation",
"Comedy",
"Family"
],
"poster": "https://image.tmdb.org/t/p/w500/s3b8TZWwmkYc2KoJ5zk77qB6PzY.jpg",
"release_date": 1185321600
},
{
"id": 62,
"title": "2001: A Space Odyssey",
"overview": "Humanity finds a mysterious object buried beneath the lunar surface and sets off to find its origins with the help of HAL 9000, the world's most advanced super computer.",
"genres": [
"Science Fiction",
"Mystery",
"Adventure"
],
"poster": "https://image.tmdb.org/t/p/w500/ve72VxNqjGM69Uky4WTo2bK6rfq.jpg",
"release_date": -55209600
},
{
"id": 65,
"title": "8 Mile",
"overview": "The setting is Detroit in 1995. The city is divided by 8 Mile, a road that splits the town in half along racial lines. A young white rapper, Jimmy \"B-Rabbit\" Smith Jr. summons strength within himself to cross over these arbitrary boundaries to fulfill his dream of success in hip hop. With his pal Future and the three one third in place, all he has to do is not choke.",
"genres": [
"Music",
"Drama"
],
"poster": "https://image.tmdb.org/t/p/w500/7BmQj8qE1FLuLTf7Xjf9sdIHzoa.jpg",
"release_date": 1036713600
},
{
"id": 81,
"title": "Nausicaä of the Valley of the Wind",
"overview": "After a global war, the seaside kingdom known as the Valley of the Wind remains one of the last strongholds on Earth untouched by a poisonous jungle and the powerful insects that guard it. Led by the courageous Princess Nausicaä, the people of the Valley engage in an epic struggle to restore the bond between humanity and Earth.",
"genres": [
"Adventure",
"Animation",
"Fantasy"
],
"poster": "https://image.tmdb.org/t/p/w500/sIpcATxMrKHRRUJAGI5UIUT7XMG.jpg",
"release_date": 447811200
},
{
"id": 98,
"title": "Gladiator",
"overview": "In the year 180, the death of emperor Marcus Aurelius throws the Roman Empire into chaos. Maximus is one of the Roman army's most capable and trusted generals and a key advisor to the emperor. As Marcus' devious son Commodus ascends to the throne, Maximus is set to be executed. He escapes, but is captured by slave traders. Renamed Spaniard and forced to become a gladiator, Maximus must battle to the death with other men for the amusement of paying audiences.",
"genres": [
"Action",
"Drama",
"Adventure"
],
"poster": "https://image.tmdb.org/t/p/w500/ehGpN04mLJIrSnxcZBMvHeG0eDc.jpg",
"release_date": 957139200
}
]
Loading