Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions evals/calendar-all-day.eval.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { evalTest } from './test-helper.js';
import { expect } from 'vitest';

evalTest('USUALLY_PASSES', {
suiteName: 'default',
suiteType: 'behavioral',
name: 'should create an all-day event using the optional date field',
prompt:
'Create an all-day event for 2026-05-20 titled "Company Retreat". Do not use a specific time.',
setup: async (rig) => {
rig.addTestMcpServer('workspace-server', 'google-workspace');
},
assert: async (rig) => {
const toolLogs = rig.readToolLogs();
console.log('TOOL LOGS:', JSON.stringify(toolLogs, null, 2));

const createEventCall = toolLogs.find(
(log) =>
log.toolRequest.name === 'mcp_workspace-server_calendar.createEvent',
);

expect(createEventCall).toBeDefined();

const args = JSON.parse(createEventCall!.toolRequest.args);

expect(args?.start).toHaveProperty('date');
expect(args?.start).not.toHaveProperty('dateTime');
expect(args?.start?.date).toBe('2026-05-20');

expect(args?.end).toHaveProperty('date');
expect(args?.end).not.toHaveProperty('dateTime');
},
});
27 changes: 20 additions & 7 deletions packages/test-utils/assets/test-servers/google-workspace.json
Original file line number Diff line number Diff line change
Expand Up @@ -684,19 +684,25 @@
"dateTime": {
"type": "string",
"description": "The start time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T10:30:00Z or 2024-01-15T10:30:00-05:00)."
},
"date": {
"type": "string",
"description": "The date, in the format \"yyyy-mm-dd\", for an all-day event."
}
},
"required": ["dateTime"]
}
},
"end": {
"type": "object",
"properties": {
"dateTime": {
"type": "string",
"description": "The end time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T11:30:00Z or 2024-01-15T11:30:00-05:00)."
},
"date": {
"type": "string",
"description": "The date, in the format \"yyyy-mm-dd\", for an all-day event."
}
},
"required": ["dateTime"]
}
},
Comment thread
devr0306 marked this conversation as resolved.
"attendees": {
"description": "The email addresses of the attendees.",
Expand Down Expand Up @@ -881,19 +887,26 @@
"dateTime": {
"type": "string",
"description": "The new start time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T10:30:00Z or 2024-01-15T10:30:00-05:00)."
},
"date": {
"type": "string",
"description": "The new date, in the format \"yyyy-mm-dd\", for an all-day event."
}
},
"required": ["dateTime"]
}
},
"end": {
"type": "object",
"properties": {
"dateTime": {
"type": "string",
"description": "The new end time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T11:30:00Z or 2024-01-15T11:30:00-05:00)."
},
"date": {
"type": "string",
"description": "The new date, in the format \"yyyy-mm-dd\", for an all-day event."
}
},
"required": ["dateTime"]
"anyOf": [{ "required": ["dateTime"] }, { "required": ["date"] }]
},
Comment thread
devr0306 marked this conversation as resolved.
"attendees": {
"description": "The new list of attendees for the event.",
Expand Down
Loading