Skip to content

chore: extend accuracy tests to support custom user and cluster config #655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 16, 2025
Merged

chore: extend accuracy tests to support custom user and cluster config #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc89c9d
chore: extend accuracy tests with vector search
himanshusinghs Oct 15, 2025
e83d9a2
chore: fix a few tests
himanshusinghs Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 135 additions & 130 deletions tests/accuracy/createIndex.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,140 +1,145 @@
import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
import { Matcher } from "./sdk/matcher.js";

// TODO: supply this with a proper config API once we refactor describeAccuracyTests to support it
process.env.MDB_VOYAGE_API_KEY = "valid-key";

describeAccuracyTests([
{
prompt: "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "classic",
keys: {
release_year: 1,
describeAccuracyTests(
[
{
prompt: "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "classic",
keys: {
release_year: 1,
},
},
},
],
],
},
},
},
],
},
{
prompt: "Create a text index on title field in 'mflix.movies' namespace",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "classic",
keys: {
title: "text",
],
},
{
prompt: "Create a text index on title field in 'mflix.movies' namespace",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "classic",
keys: {
title: "text",
},
},
},
],
],
},
},
},
],
},
{
prompt: "Create a vector search index on 'mydb.movies' namespace on the 'plotSummary' field. The index should use 1024 dimensions.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mydb",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: 1024,
},
],
},
],
],
},
{
prompt: "Create a vector search index on 'mflix.movies' namespace on the 'plotSummary' field. The index should use 1024 dimensions.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: 1024,
},
],
},
],
},
},
},
],
},
{
prompt: "Create a vector search index on 'mydb.movies' namespace with on the 'plotSummary' field and 'genre' field, both of which contain vector embeddings. Pick a sensible number of dimensions for a voyage 3.5 model.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mydb",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: Matcher.number(
(value) => value % 2 === 0 && value >= 256 && value <= 8192
),
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
},
{
type: "vector",
path: "genre",
numDimensions: Matcher.number(
(value) => value % 2 === 0 && value >= 256 && value <= 8192
),
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
},
],
},
],
],
},
{
prompt: "Create a vector search index on 'mflix.movies' namespace with on the 'plotSummary' field and 'genre' field, both of which contain vector embeddings. Pick a sensible number of dimensions for a voyage 3.5 model.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: Matcher.number(
(value) => value % 2 === 0 && value >= 256 && value <= 8192
),
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
},
{
type: "vector",
path: "genre",
numDimensions: Matcher.number(
(value) => value % 2 === 0 && value >= 256 && value <= 8192
),
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
},
],
},
],
},
},
},
],
},
{
prompt: "Create a vector search index on 'mydb.movies' namespace where the 'plotSummary' field is indexed as a 1024-dimensional vector and the 'releaseDate' field is indexed as a regular field.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mydb",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: 1024,
},
{
type: "filter",
path: "releaseDate",
},
],
},
],
],
},
{
prompt: "Create a vector search index on 'mflix.movies' namespace where the 'plotSummary' field is indexed as a 1024-dimensional vector and the 'releaseDate' field is indexed as a regular field.",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: "vectorSearch",
fields: [
{
type: "vector",
path: "plotSummary",
numDimensions: 1024,
},
{
type: "filter",
path: "releaseDate",
},
],
},
],
},
},
},
],
},
]);
],
},
],
{
userConfig: { voyageApiKey: "valid-key" },
clusterConfig: {
search: true,
},
}
);
57 changes: 57 additions & 0 deletions tests/accuracy/createIndex.vectorSearchDisabled.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/**
* Accuracy tests for when the vector search feature flag is disabled.
*
* TODO: Remove this file once we permanently enable the vector search feature.
*/
import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
import { Matcher } from "./sdk/matcher.js";

describeAccuracyTests(
[
{
prompt: "(vectorSearchDisabled) Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: Matcher.anyOf(Matcher.undefined, Matcher.value("classic")),
keys: {
release_year: 1,
},
},
],
},
},
],
},
{
prompt: "(vectorSearchDisabled) Create a text index on title field in 'mflix.movies' namespace",
expectedToolCalls: [
{
toolName: "create-index",
parameters: {
database: "mflix",
collection: "movies",
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
definition: [
{
type: Matcher.anyOf(Matcher.undefined, Matcher.value("classic")),
keys: {
title: "text",
},
},
],
},
},
],
},
],
{
userConfig: { voyageApiKey: "" },
}
);
18 changes: 7 additions & 11 deletions tests/accuracy/sdk/accuracyTestingClient.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
import { MCP_SERVER_CLI_SCRIPT } from "./constants.js";
import type { LLMToolCall } from "./accuracyResultStorage/resultStorage.js";
import type { VercelMCPClient, VercelMCPClientTools } from "./agent.js";
import type { UserConfig } from "../../../src/lib.js";

type ToolResultGeneratorFn = (parameters: Record<string, unknown>) => CallToolResult | Promise<CallToolResult>;
export type MockedTools = Record<string, ToolResultGeneratorFn>;
Expand Down Expand Up @@ -81,18 +82,13 @@ export class AccuracyTestingClient {

static async initializeClient(
mdbConnectionString: string,
atlasApiClientId?: string,
atlasApiClientSecret?: string,
voyageApiKey?: string
userConfig: Partial<{ [k in keyof UserConfig]: string }> = {}
): Promise<AccuracyTestingClient> {
const args = [
MCP_SERVER_CLI_SCRIPT,
"--connectionString",
mdbConnectionString,
...(atlasApiClientId ? ["--apiClientId", atlasApiClientId] : []),
...(atlasApiClientSecret ? ["--apiClientSecret", atlasApiClientSecret] : []),
...(voyageApiKey ? ["--voyageApiKey", voyageApiKey] : []),
];
const additionalArgs = Object.entries(userConfig).flatMap(([key, value]) => {
return [`--${key}`, value];
});

const args = [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString, ...additionalArgs];

const clientTransport = new StdioClientTransport({
command: process.execPath,
Expand Down
Loading
Loading