diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml index b43b80b0..309ec3d6 100644 --- a/.code-samples.meilisearch.yaml +++ b/.code-samples.meilisearch.yaml @@ -808,3 +808,18 @@ update_embedders_1: |- 'documentTemplate' => 'A document titled '{{doc.title}}' whose description starts with {{doc.overview|truncatewords: 20}}' ] ]); +search_parameter_reference_media_1: |- + $client->index('INDEX_NAME')->search('a futuristic movie', [ + 'hybrid' => [ + 'embedder' => 'EMBEDDER_NAME' + ], + 'media' => [ + 'textAndPoster' => [ + 'text' => 'a futuristic movie', + 'image' => [ + 'mime' => 'image/jpeg', + 'data' => 'base64EncodedImageData' + ] + ] + ] + ]); diff --git a/tests/Endpoints/MultiModalSearchTest.php b/tests/Endpoints/MultiModalSearchTest.php new file mode 100644 index 00000000..0e7d329d --- /dev/null +++ b/tests/Endpoints/MultiModalSearchTest.php @@ -0,0 +1,196 @@ +host, getenv('MEILISEARCH_API_KEY')); + $http->patch('/experimental-features', ['multimodal' => true]); + + $voyageApiKey = getenv('VOYAGE_API_KEY'); + if (false === $voyageApiKey || '' === $voyageApiKey) { + self::markTestSkipped('Missing `VOYAGE_API_KEY` environment variable'); + } + + $this->index = $this->createEmptyIndex($this->safeIndexName()); + $updateSettingsPromise = $this->index->updateSettings([ + 'searchableAttributes' => ['title', 'overview'], + 'embedders' => [ + 'multimodal' => self::getVoyageEmbedderConfig($voyageApiKey), + ], + ]); + $this->index->waitForTask($updateSettingsPromise['taskUid']); + + // Load the movies.json dataset + $documentsJson = file_get_contents('./tests/datasets/movies.json'); + $this->documents = json_decode($documentsJson, true, 512, JSON_THROW_ON_ERROR); + $addDocumentsPromise = $this->index->addDocuments($this->documents); + $this->index->waitForTask($addDocumentsPromise['taskUid']); + } + + public function testTextOnlySearch(): void + { + $query = 'A movie with lightsabers in space'; + $response = $this->index->search($query, [ + 'media' => [ + 'text' => ['text' => $query], + ], + 'hybrid' => [ + 'embedder' => 'multimodal', + 'semanticRatio' => 1, + ], + ]); + self::assertSame('Star Wars', $response->getHits()[0]['title']); + } + + public function testImageOnlySearch(): void + { + $theFifthElementPoster = $this->documents[3]['poster']; + $response = $this->index->search(null, [ + 'media' => [ + 'poster' => [ + 'poster' => $theFifthElementPoster, + ], + ], + 'hybrid' => [ + 'embedder' => 'multimodal', + 'semanticRatio' => 1, + ], + ]); + self::assertSame('The Fifth Element', $response->getHits()[0]['title']); + } + + public function testTextAndImageSearch(): void + { + $query = 'a futuristic movie'; + $masterYodaBase64 = base64_encode(file_get_contents('./tests/assets/master-yoda.jpeg')); + $response = $this->index->search(null, [ + 'media' => [ + 'textAndPoster' => [ + 'text' => $query, + 'image' => [ + 'mime' => 'image/jpeg', + 'data' => $masterYodaBase64, + ], + ], + ], + 'hybrid' => [ + 'embedder' => 'multimodal', + 'semanticRatio' => 1, + ], + ]); + self::assertSame('Star Wars', $response->getHits()[0]['title']); + } + + private static function getVoyageEmbedderConfig(string $voyageApiKey): array + { + return [ + 'source' => 'rest', + 'url' => 'https://api.voyageai.com/v1/multimodalembeddings', + 'apiKey' => $voyageApiKey, + 'dimensions' => 1024, + 'indexingFragments' => [ + 'textAndPoster' => [ + // the shape of the data here depends on the model used + 'value' => [ + 'content' => [ + [ + 'type' => 'text', + 'text' => 'A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.', + ], + [ + 'type' => 'image_url', + 'image_url' => '{{doc.poster}}', + ], + ], + ], + ], + 'text' => [ + 'value' => [ + // The shape of the data here depends on the model used + 'content' => [ + [ + 'type' => 'text', + 'text' => 'A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.', + ], + ], + ], + ], + 'poster' => [ + 'value' => [ + // The shape of the data here depends on the model used + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => '{{doc.poster}}', + ], + ], + ], + ], + ], + 'searchFragments' => [ + 'textAndPoster' => [ + 'value' => [ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{{media.textAndPoster.text}}', + ], + [ + 'type' => 'image_base64', + 'image_base64' => 'data:{{media.textAndPoster.image.mime}};base64,{{media.textAndPoster.image.data}}', + ], + ], + ], + ], + 'text' => [ + 'value' => [ + 'content' => [ + [ + 'type' => 'text', + 'text' => '{{media.text.text}}', + ], + ], + ], + ], + 'poster' => [ + 'value' => [ + 'content' => [ + [ + 'type' => 'image_url', + 'image_url' => '{{media.poster.poster}}', + ], + ], + ], + ], + ], + 'request' => [ + // This request object matches the Voyage API request object + 'inputs' => ['{{fragment}}', '{{..}}'], + 'model' => 'voyage-multimodal-3', + ], + 'response' => [ + // This response object matches the Voyage API response object + 'data' => [ + [ + 'embedding' => '{{embedding}}', + ], + '{{..}}', + ], + ], + ]; + } +} diff --git a/tests/assets/master-yoda.jpeg b/tests/assets/master-yoda.jpeg new file mode 100644 index 00000000..f241a85a Binary files /dev/null and b/tests/assets/master-yoda.jpeg differ diff --git a/tests/datasets/movies.json b/tests/datasets/movies.json new file mode 100644 index 00000000..1f89aaa7 --- /dev/null +++ b/tests/datasets/movies.json @@ -0,0 +1,133 @@ +[ + { + "id": 11, + "title": "Star Wars", + "overview": "Princess Leia is captured and held hostage by the evil Imperial forces in their effort to take over the galactic Empire. Venturesome Luke Skywalker and dashing captain Han Solo team together with the loveable robot duo R2-D2 and C-3PO to rescue the beautiful princess and restore peace and justice in the Empire.", + "genres": [ + "Adventure", + "Action", + "Science Fiction" + ], + "poster": "https://image.tmdb.org/t/p/w500/6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg", + "release_date": 233366400 + }, + { + "id": 12, + "title": "Finding Nemo", + "overview": "Nemo, an adventurous young clownfish, is unexpectedly taken from his Great Barrier Reef home to a dentist's office aquarium. It's up to his worrisome father Marlin and a friendly but forgetful fish Dory to bring Nemo home -- meeting vegetarian sharks, surfer dude turtles, hypnotic jellyfish, hungry seagulls, and more along the way.", + "genres": [ + "Animation", + "Family" + ], + "poster": "https://image.tmdb.org/t/p/w500/eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg", + "release_date": 1054252800 + }, + { + "id": 13, + "title": "Forrest Gump", + "overview": "A man with a low IQ has accomplished great things in his life and been present during significant historic events—in each case, far exceeding what anyone imagined he could do. But despite all he has achieved, his one true love eludes him.", + "genres": [ + "Comedy", + "Drama", + "Romance" + ], + "poster": "https://image.tmdb.org/t/p/w500/h5J4W4veyxMXDMjeNxZI46TsHOb.jpg", + "release_date": 773452800 + }, + { + "id": 18, + "title": "The Fifth Element", + "overview": "In 2257, a taxi driver is unintentionally given the task of saving a young girl who is part of the key that will ensure the survival of humanity.", + "genres": [ + "Adventure", + "Fantasy", + "Action", + "Thriller", + "Science Fiction" + ], + "poster": "https://image.tmdb.org/t/p/w500/fPtlCO1yQtnoLHOwKtWz7db6RGU.jpg", + "release_date": 862531200 + }, + { + "id": 22, + "title": "Pirates of the Caribbean: The Curse of the Black Pearl", + "overview": "Jack Sparrow, a freewheeling 18th-century pirate, quarrels with a rival pirate bent on pillaging Port Royal. When the governor's daughter is kidnapped, Sparrow decides to help the girl's love save her.", + "genres": [ + "Adventure", + "Fantasy", + "Action" + ], + "poster": "https://image.tmdb.org/t/p/w500/z8onk7LV9Mmw6zKz4hT6pzzvmvl.jpg", + "release_date": 1057708800 + }, + { + "id": 24, + "title": "Kill Bill: Vol. 1", + "overview": "An assassin is shot by her ruthless employer, Bill, and other members of their assassination circle – but she lives to plot her vengeance.", + "genres": [ + "Action", + "Crime" + ], + "poster": "https://image.tmdb.org/t/p/w500/v7TaX8kXMXs5yFFGR41guUDNcnB.jpg", + "release_date": 1065744000 + }, + { + "id": 35, + "title": "The Simpsons Movie", + "overview": "After Homer accidentally pollutes the town's water supply, Springfield is encased in a gigantic dome by the EPA and the Simpsons are declared fugitives.", + "genres": [ + "Animation", + "Comedy", + "Family" + ], + "poster": "https://image.tmdb.org/t/p/w500/s3b8TZWwmkYc2KoJ5zk77qB6PzY.jpg", + "release_date": 1185321600 + }, + { + "id": 62, + "title": "2001: A Space Odyssey", + "overview": "Humanity finds a mysterious object buried beneath the lunar surface and sets off to find its origins with the help of HAL 9000, the world's most advanced super computer.", + "genres": [ + "Science Fiction", + "Mystery", + "Adventure" + ], + "poster": "https://image.tmdb.org/t/p/w500/ve72VxNqjGM69Uky4WTo2bK6rfq.jpg", + "release_date": -55209600 + }, + { + "id": 65, + "title": "8 Mile", + "overview": "The setting is Detroit in 1995. The city is divided by 8 Mile, a road that splits the town in half along racial lines. A young white rapper, Jimmy \"B-Rabbit\" Smith Jr. summons strength within himself to cross over these arbitrary boundaries to fulfill his dream of success in hip hop. With his pal Future and the three one third in place, all he has to do is not choke.", + "genres": [ + "Music", + "Drama" + ], + "poster": "https://image.tmdb.org/t/p/w500/7BmQj8qE1FLuLTf7Xjf9sdIHzoa.jpg", + "release_date": 1036713600 + }, + { + "id": 81, + "title": "Nausicaä of the Valley of the Wind", + "overview": "After a global war, the seaside kingdom known as the Valley of the Wind remains one of the last strongholds on Earth untouched by a poisonous jungle and the powerful insects that guard it. Led by the courageous Princess Nausicaä, the people of the Valley engage in an epic struggle to restore the bond between humanity and Earth.", + "genres": [ + "Adventure", + "Animation", + "Fantasy" + ], + "poster": "https://image.tmdb.org/t/p/w500/sIpcATxMrKHRRUJAGI5UIUT7XMG.jpg", + "release_date": 447811200 + }, + { + "id": 98, + "title": "Gladiator", + "overview": "In the year 180, the death of emperor Marcus Aurelius throws the Roman Empire into chaos. Maximus is one of the Roman army's most capable and trusted generals and a key advisor to the emperor. As Marcus' devious son Commodus ascends to the throne, Maximus is set to be executed. He escapes, but is captured by slave traders. Renamed Spaniard and forced to become a gladiator, Maximus must battle to the death with other men for the amusement of paying audiences.", + "genres": [ + "Action", + "Drama", + "Adventure" + ], + "poster": "https://image.tmdb.org/t/p/w500/ehGpN04mLJIrSnxcZBMvHeG0eDc.jpg", + "release_date": 957139200 + } +]