Bare metal evals fixes

apps/web-evals/package.json

@@ -5,7 +5,7 @@
 	"scripts": {
 		"lint": "next lint --max-warnings 0",
 		"check-types": "tsc -b",
-		"dev": "scripts/check-services.sh && next dev",
+		"dev": "scripts/check-services.sh && next dev -p 3446",
 		"format": "prettier --write src",
 		"build": "next build",
 		"start": "next start",

packages/evals/README.md

@@ -95,7 +95,7 @@ By default, the evals system uses the following ports:
 
 - **PostgreSQL**: 5433 (external) → 5432 (internal)
 - **Redis**: 6380 (external) → 6379 (internal)
-- **Web Service**: 3446 (external) → 3000 (internal)
+- **Web Service**: 3446 (external) → 3446 (internal)
 
 These ports are configured to avoid conflicts with other services that might be running on the standard PostgreSQL (5432) and Redis (6379) ports.
 

packages/evals/docker-compose.yml

@@ -52,7 +52,7 @@ services:
             context: ../../
             dockerfile: packages/evals/Dockerfile.web
         ports:
-            - "${EVALS_WEB_PORT:-3446}:3000"
+            - "${EVALS_WEB_PORT:-3446}:3446"
         environment:
             - HOST_EXECUTION_METHOD=docker
         volumes:

packages/evals/scripts/setup.sh

@@ -12,7 +12,6 @@ build_extension() {
   echo "🔨 Building the Roo Code extension..."
   pnpm -w vsix -- --out ../bin/roo-code-$(git rev-parse --short HEAD).vsix || exit 1
   code --install-extension ../../bin/roo-code-$(git rev-parse --short HEAD).vsix || exit 1
-  cd evals
 }
 
 check_docker_services() {
@@ -377,7 +376,7 @@ fi
 
 echo -e "\n🚀 You're ready to rock and roll! \n"
 
-if ! nc -z localhost 3000; then
+if ! nc -z localhost 3446; then
   read -p "🌐 Would you like to start the evals web app? (Y/n): " start_evals
 
   if [[ "$start_evals" =~ ^[Yy]|^$ ]]; then
@@ -386,5 +385,5 @@ if ! nc -z localhost 3000; then
     echo "💡 You can start it anytime with 'pnpm --filter @roo-code/web-evals dev'."
   fi
 else
-  echo "👟 The evals web app is running at http://localhost:3000 (or http://localhost:3446 if using Docker)"
+  echo "👟 The evals web app is running at http://localhost:3446"
 fi

src/api/providers/roo.ts

@@ -1,22 +1,22 @@
 import { Anthropic } from "@anthropic-ai/sdk"
+import OpenAI from "openai"
 
-import { rooDefaultModelId, rooModels, type RooModelId } from "@roo-code/types"
+import { AuthState, rooDefaultModelId, rooModels, type RooModelId } from "@roo-code/types"
 import { CloudService } from "@roo-code/cloud"
 
 import type { ApiHandlerOptions } from "../../shared/api"
 import { ApiStream } from "../transform/stream"
 
 import type { ApiHandlerCreateMessageMetadata } from "../index"
+import { DEFAULT_HEADERS } from "./constants"
 import { BaseOpenAiCompatibleProvider } from "./base-openai-compatible-provider"
 
 export class RooHandler extends BaseOpenAiCompatibleProvider<RooModelId> {
 	constructor(options: ApiHandlerOptions) {
-		// Get the session token if available, but don't throw if not.
-		// The server will handle authentication errors and return appropriate status codes.
-		let sessionToken = ""
+		let sessionToken: string | undefined = undefined
 
 		if (CloudService.hasInstance()) {
-			sessionToken = CloudService.instance.authService?.getSessionToken() || ""
+			sessionToken = CloudService.instance.authService?.getSessionToken()
 		}
 
 		// Always construct the handler, even without a valid token.
@@ -25,11 +25,25 @@ export class RooHandler extends BaseOpenAiCompatibleProvider<RooModelId> {
 			...options,
 			providerName: "Roo Code Cloud",
 			baseURL: process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy/v1",
-			apiKey: sessionToken || "unauthenticated", // Use a placeholder if no token
+			apiKey: sessionToken || "unauthenticated", // Use a placeholder if no token.
 			defaultProviderModelId: rooDefaultModelId,
 			providerModels: rooModels,
 			defaultTemperature: 0.7,
 		})
+
+		if (CloudService.hasInstance()) {
+			const cloudService = CloudService.instance
+
+			cloudService.on("auth-state-changed", (state: { state: AuthState }) => {
+				if (state.state === "active-session") {
+					this.client = new OpenAI({
+						baseURL: this.baseURL,
+						apiKey: this.options.apiKey,
+						defaultHeaders: DEFAULT_HEADERS,
+					})
+				}
+			})
+		}
 	}
 
 	override async *createMessage(

src/extension.ts

@@ -194,7 +194,7 @@ export async function activate(context: vscode.ExtensionContext) {
 	// Add to subscriptions for proper cleanup on deactivate.
 	context.subscriptions.push(cloudService)
 
-	// Trigger initial cloud profile sync now that CloudService is ready
+	// Trigger initial cloud profile sync now that CloudService is ready.
 	try {
 		await provider.initializeCloudProfileSyncWhenReady()
 	} catch (error) {