feat: add image generation tool with OpenRouter integration (#7474)

Co-authored-by: Matt Rubens <[email protected]>
Co-authored-by: cte <[email protected]>

Author: 3 people

packages/types/npm/package.metadata.json

@@ -1,6 +1,6 @@
 {
 	"name": "@roo-code/types",
-	"version": "1.74.0",
+	"version": "1.63.0",
 	"description": "TypeScript type definitions for Roo Code.",
 	"publishConfig": {
 		"access": "public",

packages/types/src/experiment.ts

@@ -11,7 +11,6 @@ export const experimentIds = [
 	"multiFileApplyDiff",
 	"preventFocusDisruption",
 	"imageGeneration",
-	"runSlashCommand",
 ] as const
 
 export const experimentIdsSchema = z.enum(experimentIds)
@@ -27,7 +26,6 @@ export const experimentsSchema = z.object({
 	multiFileApplyDiff: z.boolean().optional(),
 	preventFocusDisruption: z.boolean().optional(),
 	imageGeneration: z.boolean().optional(),
-	runSlashCommand: z.boolean().optional(),
 })
 
 export type Experiments = z.infer<typeof experimentsSchema>

packages/types/src/provider-settings.ts

@@ -143,6 +143,13 @@ const openRouterSchema = baseProviderSettingsSchema.extend({
 	openRouterBaseUrl: z.string().optional(),
 	openRouterSpecificProvider: z.string().optional(),
 	openRouterUseMiddleOutTransform: z.boolean().optional(),
+	// Image generation settings (experimental)
+	openRouterImageGenerationSettings: z
+		.object({
+			openRouterApiKey: z.string().optional(),
+			selectedModel: z.string().optional(),
+		})
+		.optional(),
 })
 
 const bedrockSchema = apiModelIdProviderModelSchema.extend({

packages/types/src/tool.ts

@@ -34,7 +34,6 @@ export const toolNames = [
 	"fetch_instructions",
 	"codebase_search",
 	"update_todo_list",
-	"run_slash_command",
 	"generate_image",
 ] as const
 

pnpm-lock.yaml

@@ -25,7 +25,6 @@ import { getModelEndpoints } from "./fetchers/modelEndpointCache"
 import { DEFAULT_HEADERS } from "./constants"
 import { BaseProvider } from "./base-provider"
 import type { SingleCompletionHandler } from "../index"
-import { handleOpenAIError } from "./utils/openai-error-handler"
 
 // Image generation types
 interface ImageGenerationResponse {
@@ -86,7 +85,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 	private client: OpenAI
 	protected models: ModelRecord = {}
 	protected endpoints: ModelRecord = {}
-	private readonly providerName = "OpenRouter"
 
 	constructor(options: ApiHandlerOptions) {
 		super()
@@ -163,12 +161,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			...(reasoning && { reasoning }),
 		}
 
-		let stream
-		try {
-			stream = await this.client.chat.completions.create(completionParams)
-		} catch (error) {
-			throw handleOpenAIError(error, this.providerName)
-		}
+		const stream = await this.client.chat.completions.create(completionParams)
 
 		let lastUsage: CompletionUsage | undefined = undefined
 
@@ -266,12 +259,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			...(reasoning && { reasoning }),
 		}
 
-		let response
-		try {
-			response = await this.client.chat.completions.create(completionParams)
-		} catch (error) {
-			throw handleOpenAIError(error, this.providerName)
-		}
+		const response = await this.client.chat.completions.create(completionParams)
 
 		if ("error" in response) {
 			const error = response.error as { message?: string; code?: number }
@@ -287,15 +275,9 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 	 * @param prompt The text prompt for image generation
 	 * @param model The model to use for generation
 	 * @param apiKey The OpenRouter API key (must be explicitly provided)
-	 * @param inputImage Optional base64 encoded input image data URL
 	 * @returns The generated image data and format, or an error
 	 */
-	async generateImage(
-		prompt: string,
-		model: string,
-		apiKey: string,
-		inputImage?: string,
-	): Promise<ImageGenerationResult> {
+	async generateImage(prompt: string, model: string, apiKey: string): Promise<ImageGenerationResult> {
 		if (!apiKey) {
 			return {
 				success: false,
@@ -317,20 +299,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 					messages: [
 						{
 							role: "user",
-							content: inputImage
-								? [
-										{
-											type: "text",
-											text: prompt,
-										},
-										{
-											type: "image_url",
-											image_url: {
-												url: inputImage,
-											},
-										},
-									]
-								: prompt,
+							content: prompt,
 						},
 					],
 					modalities: ["image", "text"],

src/api/providers/openrouter.ts

@@ -28,7 +28,6 @@ import { attemptCompletionTool } from "../tools/attemptCompletionTool"
 import { newTaskTool } from "../tools/newTaskTool"
 
 import { updateTodoListTool } from "../tools/updateTodoListTool"
-import { runSlashCommandTool } from "../tools/runSlashCommandTool"
 import { generateImageTool } from "../tools/generateImageTool"
 
 import { formatResponse } from "../prompts/responses"
@@ -224,8 +223,6 @@ export async function presentAssistantMessage(cline: Task) {
 						const modeName = getModeBySlug(mode, customModes)?.name ?? mode
 						return `[${block.name} in ${modeName} mode: '${message}']`
 					}
-					case "run_slash_command":
-						return `[${block.name} for '${block.params.command}'${block.params.args ? ` with args: ${block.params.args}` : ""}]`
 					case "generate_image":
 						return `[${block.name} for '${block.params.path}']`
 				}
@@ -558,9 +555,6 @@ export async function presentAssistantMessage(cline: Task) {
 						askFinishSubTaskApproval,
 					)
 					break
-				case "run_slash_command":
-					await runSlashCommandTool(cline, block, askApproval, handleError, pushToolResult, removeClosingTag)
-					break
 				case "generate_image":
 					await generateImageTool(cline, block, askApproval, handleError, pushToolResult, removeClosingTag)
 					break

src/core/assistant-message/presentAssistantMessage.ts

@@ -2,35 +2,19 @@ import { ToolArgs } from "./types"
 
 export function getGenerateImageDescription(args: ToolArgs): string {
 	return `## generate_image
-Description: Request to generate or edit an image using AI models through OpenRouter API. This tool can create new images from text prompts or modify existing images based on your instructions. When an input image is provided, the AI will apply the requested edits, transformations, or enhancements to that image.
+Description: Request to generate an image using AI models through OpenRouter API. This tool creates images from text prompts and saves them to the specified path.
 Parameters:
-- prompt: (required) The text prompt describing what to generate or how to edit the image
-- path: (required) The file path where the generated/edited image should be saved (relative to the current workspace directory ${args.cwd}). The tool will automatically add the appropriate image extension if not provided.
-- image: (optional) The file path to an input image to edit or transform (relative to the current workspace directory ${args.cwd}). Supported formats: PNG, JPG, JPEG, GIF, WEBP.
+- prompt: (required) The text prompt describing the image to generate
+- path: (required) The file path where the generated image should be saved (relative to the current workspace directory ${args.cwd}). The tool will automatically add the appropriate image extension if not provided.
 Usage:
 <generate_image>
 <prompt>Your image description here</prompt>
 <path>path/to/save/image.png</path>
-<image>path/to/input/image.jpg</image>
 </generate_image>
 
 Example: Requesting to generate a sunset image
 <generate_image>
 <prompt>A beautiful sunset over mountains with vibrant orange and purple colors</prompt>
 <path>images/sunset.png</path>
-</generate_image>
-
-Example: Editing an existing image
-<generate_image>
-<prompt>Transform this image into a watercolor painting style</prompt>
-<path>images/watercolor-output.png</path>
-<image>images/original-photo.jpg</image>
-</generate_image>
-
-Example: Upscaling and enhancing an image
-<generate_image>
-<prompt>Upscale this image to higher resolution, enhance details, improve clarity and sharpness while maintaining the original content and composition</prompt>
-<path>images/enhanced-photo.png</path>
-<image>images/low-res-photo.jpg</image>
 </generate_image>`
 }

src/core/prompts/tools/generate-image.ts

@@ -25,7 +25,6 @@ import { getSwitchModeDescription } from "./switch-mode"
 import { getNewTaskDescription } from "./new-task"
 import { getCodebaseSearchDescription } from "./codebase-search"
 import { getUpdateTodoListDescription } from "./update-todo-list"
-import { getRunSlashCommandDescription } from "./run-slash-command"
 import { getGenerateImageDescription } from "./generate-image"
 import { CodeIndexManager } from "../../../services/code-index/manager"
 
@@ -58,7 +57,6 @@ const toolDescriptionMap: Record<string, (args: ToolArgs) => string | undefined>
 	apply_diff: (args) =>
 		args.diffStrategy ? args.diffStrategy.getToolDescription({ cwd: args.cwd, toolOptions: args.toolOptions }) : "",
 	update_todo_list: (args) => getUpdateTodoListDescription(args),
-	run_slash_command: () => getRunSlashCommandDescription(),
 	generate_image: (args) => getGenerateImageDescription(args),
 }
 
@@ -138,11 +136,6 @@ export function getToolDescriptionsForMode(
 		tools.delete("generate_image")
 	}
 
-	// Conditionally exclude run_slash_command if experiment is not enabled
-	if (!experiments?.runSlashCommand) {
-		tools.delete("run_slash_command")
-	}
-
 	// Map tool descriptions for allowed tools
 	const descriptions = Array.from(tools).map((toolName) => {
 		const descriptionFn = toolDescriptionMap[toolName]
@@ -178,6 +171,5 @@ export {
 	getInsertContentDescription,
 	getSearchAndReplaceDescription,
 	getCodebaseSearchDescription,
-	getRunSlashCommandDescription,
 	getGenerateImageDescription,
 }