diff --git a/.eslintignore b/.eslintignore index dab3af32662..03de6bf3636 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,5 +2,4 @@ rust/**/*.ts !rust/kcl-language-server/client/src/**/*.ts *.typegen.ts packages/codemirror-lsp-client/dist/* -e2e/playwright/snapshots/prompt-to-edit/* .vscode-test diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 846e211c41d..b9b26db7bcb 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -246,7 +246,7 @@ jobs: shell: bash id: git-check run: | - git add e2e/playwright/snapshot-tests.spec.ts-snapshots e2e/playwright/snapshots + git add e2e/playwright/snapshot-tests.spec.ts-snapshots if git status | grep -q "Changes to be committed" then echo "modified=true" >> $GITHUB_OUTPUT else echo "modified=false" >> $GITHUB_OUTPUT @@ -257,7 +257,7 @@ jobs: if: ${{ false && needs.conditions.outputs.should-run == 'true' && steps.git-check.outputs.modified == 'true' }} shell: bash run: | - git add e2e/playwright/snapshot-tests.spec.ts-snapshots e2e/playwright/snapshots + git add e2e/playwright/snapshot-tests.spec.ts-snapshots git config --local user.email "github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" git remote set-url origin https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git diff --git a/.prettierignore b/.prettierignore index 4dea6fe02b4..b2927859bcb 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,8 +8,6 @@ coverage *.hbs target e2e/playwright/export-snapshots -e2e/playwright/snapshots/prompt-to-edit - # XState generated files src/machines/**.typegen.ts diff --git a/e2e/playwright/feature-tree-pane.spec.ts b/e2e/playwright/feature-tree-pane.spec.ts index 019f6c9e440..673b02cedfc 100644 --- a/e2e/playwright/feature-tree-pane.spec.ts +++ b/e2e/playwright/feature-tree-pane.spec.ts @@ -1,6 +1,10 @@ import * as fsp from 'fs/promises' import { join } from 'path' +import { + orRunWhenFullSuiteEnabled, + runningOnWindows, +} from '@e2e/playwright/test-utils' import { expect, test } from '@e2e/playwright/zoo-test' const FEATURE_TREE_EXAMPLE_CODE = `export fn timesFive(x) { @@ -65,6 +69,9 @@ test.describe('Feature Tree pane', () => { 'User can go to definition and go to function definition', { tag: '@electron' }, async ({ context, homePage, scene, editor, toolbar, cmdBar, page }) => { + if (runningOnWindows()) { + test.fixme(orRunWhenFullSuiteEnabled()) + } await context.folderSetupFn(async (dir) => { const bracketDir = join(dir, 'test-sample') await fsp.mkdir(bracketDir, { recursive: true }) diff --git a/e2e/playwright/fixtures/cmdBarFixture.ts b/e2e/playwright/fixtures/cmdBarFixture.ts index 8754f879ba3..b4461e464d7 100644 --- a/e2e/playwright/fixtures/cmdBarFixture.ts +++ b/e2e/playwright/fixtures/cmdBarFixture.ts @@ -1,7 +1,5 @@ -import type { Locator, Page, Request, Route, TestInfo } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' import { expect } from '@playwright/test' -import * as fs from 'fs' -import * as path from 'path' type CmdBarSerialised = | { @@ -177,71 +175,4 @@ export class CmdBarFixture { selectOption = (options: Parameters[1]) => { return this.page.getByRole('option', options) } - - /** - * Captures a snapshot of the request sent to the text-to-cad API endpoint - * and saves it to a file named after the current test. - * - * The snapshot file will be saved in the specified directory with a filename - * derived from the test's full path (including describe blocks). - * - * @param testInfoInOrderToGetTestTitle The TestInfo object from the test context - * @param customOutputDir Optional custom directory for the output file - */ - async captureTextToCadRequestSnapshot( - testInfoInOrderToGetTestTitle: TestInfo, - customOutputDir = 'e2e/playwright/snapshots/prompt-to-edit' - ) { - // First sanitize each title component individually - const sanitizedTitleComponents = [ - ...testInfoInOrderToGetTestTitle.titlePath.slice(0, -1), // Get all parent titles - testInfoInOrderToGetTestTitle.title, // Add the test title - ].map( - (component) => - component - .replace(/[^a-z0-9]/gi, '-') // Replace non-alphanumeric chars with hyphens - .toLowerCase() - .replace(/-+/g, '-') // Replace multiple consecutive hyphens with a single one - .replace(/^-|-$/g, '') // Remove leading/trailing hyphens - ) - - // Join the sanitized components with -- as a clear separator - const sanitizedTestName = sanitizedTitleComponents.join('--') - - // Create the output path - const outputPath = path.join( - customOutputDir, - `${sanitizedTestName}.snap.json` - ) - - // Create a handler function that saves request bodies to a file - const requestHandler = (route: Route, request: Request) => { - try { - const requestBody = request.postDataJSON() - - // Ensure directory exists - const dir = path.dirname(outputPath) - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }) - } - - // Write the request body to the file - fs.writeFileSync(outputPath, JSON.stringify(requestBody, null, 2)) - - console.log(`Saved text-to-cad API request to: ${outputPath}`) - } catch (error) { - console.error('Error processing text-to-cad request:', error) - } - - // Use void to explicitly mark the promise as ignored - void route.continue() - } - - // Start monitoring requests - await this.page.route('**/ml/text-to-cad/iteration', requestHandler) - - console.log( - `Monitoring text-to-cad API requests. Output will be saved to: ${outputPath}` - ) - } } diff --git a/e2e/playwright/prompt-to-edit-snapshot-tests.spec.ts b/e2e/playwright/prompt-to-edit-snapshot-tests.spec.ts deleted file mode 100644 index 84bd40736d5..00000000000 --- a/e2e/playwright/prompt-to-edit-snapshot-tests.spec.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { expect, test } from '@e2e/playwright/zoo-test' - -/* eslint-disable jest/no-conditional-expect */ - -/** - * Snapshot Tests for Text-to-CAD API Requests - * - * These tests are primarily designed to capture the requests sent to the Text-to-CAD API - * rather than to verify application behavior. Unlike regular tests, these tests: - * - * 1. Don't assert much about the application's response or state changes - * 2. Focus on setting up specific scenarios and triggering API requests - * 3. Use the captureTextToCadRequestSnapshot() method to save request payloads to snapshot files - * - * The main purpose is to maintain a collection of real-world API request examples that can be: - * - Used for regression testing the (AI) API - * - Referenced when making changes to the Text-to-CAD integration, particularly the meta-prompts - * the frontend adds to the user's prompt - * - * These tests intentionally don't wait for or verify responses, as we're primarily - * interested in capturing the outgoing requests for documentation and analysis. - * - */ - -const file = `sketch001 = startSketchOn(XZ) -profile001 = startProfileAt([57.81, 250.51], sketch001) - |> line(end = [121.13, 56.63], tag = $seg02) - |> line(end = [83.37, -34.61], tag = $seg01) - |> line(end = [19.66, -116.4]) - |> line(end = [-221.8, -41.69]) - |> line(endAbsolute = [profileStartX(%), profileStartY(%)]) - |> close() -extrude001 = extrude(profile001, length = 200) -sketch002 = startSketchOn(XZ) - |> startProfileAt([-73.64, -42.89], %) - |> xLine(length = 173.71) - |> line(end = [-22.12, -94.4]) - |> xLine(length = -156.98) - |> line(endAbsolute = [profileStartX(%), profileStartY(%)]) - |> close() -extrude002 = extrude(sketch002, length = 50) -sketch003 = startSketchOn(XY) - |> startProfileAt([52.92, 157.81], %) - |> angledLine([0, 176.4], %, $rectangleSegmentA001) - |> angledLine([ - segAng(rectangleSegmentA001) - 90, - 53.4 - ], %, $rectangleSegmentB001) - |> angledLine([ - segAng(rectangleSegmentA001), - -segLen(rectangleSegmentA001) - ], %, $rectangleSegmentC001) - |> line(endAbsolute = [profileStartX(%), profileStartY(%)]) - |> close() -extrude003 = extrude(sketch003, length = 20) -` -test.describe('edit with AI example snapshots', () => { - test( - `change colour`, - { tag: '@snapshot' }, - async ({ context, homePage, cmdBar, editor, page, scene }) => { - await context.addInitScript((file) => { - localStorage.setItem('persistCode', file) - }, file) - await homePage.goToModelingScene() - await scene.settled(cmdBar) - - const body1CapCoords = { x: 571, y: 351 } - const [clickBody1Cap] = scene.makeMouseHelpers( - body1CapCoords.x, - body1CapCoords.y - ) - const yellow: [number, number, number] = [179, 179, 131] - const submittingToast = page.getByText('Submitting to Text-to-CAD API...') - - await test.step('wait for scene to load select body and check selection came through', async () => { - await scene.expectPixelColor([134, 134, 134], body1CapCoords, 15) - await clickBody1Cap() - await scene.expectPixelColor(yellow, body1CapCoords, 20) - await editor.expectState({ - highlightedCode: '', - activeLines: ['|>startProfileAt([-73.64,-42.89],%)'], - diagnostics: [], - }) - }) - - await test.step('fire off edit prompt', async () => { - await cmdBar.captureTextToCadRequestSnapshot(test.info()) - await cmdBar.openCmdBar('promptToEdit') - // being specific about the color with a hex means asserting pixel color is more stable - await page - .getByTestId('cmd-bar-arg-value') - .fill('make this neon green please, use #39FF14') - await page.waitForTimeout(100) - await cmdBar.progressCmdBar() - await expect(submittingToast).toBeVisible() - }) - } - ) -}) diff --git a/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--change-colour.snap.json b/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--change-colour.snap.json deleted file mode 100644 index 00de798d1b7..00000000000 --- a/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--change-colour.snap.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "original_source_code": "sketch001 = startSketchOn('XZ')\nprofile001 = startProfileAt([57.81, 250.51], sketch001)\n |> line(end = [121.13, 56.63], tag = $seg02)\n |> line(end = [83.37, -34.61], tag = $seg01)\n |> line(end = [19.66, -116.4])\n |> line(end = [-221.8, -41.69])\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude001 = extrude(profile001, length = 200)\nsketch002 = startSketchOn('XZ')\n |> startProfileAt([-73.64, -42.89], %)\n |> xLine(length = 173.71)\n |> line(end = [-22.12, -94.4])\n |> xLine(length = -156.98)\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude002 = extrude(sketch002, length = 50)\nsketch003 = startSketchOn(XY)\n |> startProfileAt([52.92, 157.81], %)\n |> angledLine([0, 176.4], %, $rectangleSegmentA001)\n |> angledLine([\n segAng(rectangleSegmentA001) - 90,\n 53.4\n ], %, $rectangleSegmentB001)\n |> angledLine([\n segAng(rectangleSegmentA001),\n -segLen(rectangleSegmentA001)\n ], %, $rectangleSegmentC001)\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude003 = extrude(sketch003, length = 20)\n", - "prompt": "make this neon green please, use #39FF14", - "source_ranges": [ - { - "prompt": "The users main selection is the end cap of a general-sweep (that is an extrusion, revolve, sweep or loft).\nThe source range most likely refers to \"startProfileAt\" simply because this is the start of the profile that was swept.\nIf you need to operate on this cap, for example for sketching on the face, you can use the special string END i.e. `startSketchOn(someSweepVariable, END)`\nWhen they made this selection they main have intended this surface directly or meant something more general like the sweep body.\nSee later source ranges for more context.", - "range": { - "start": { - "line": 11, - "column": 5 - }, - "end": { - "line": 11, - "column": 40 - } - } - }, - { - "prompt": "This is the sweep's source range from the user's main selection of the end cap.", - "range": { - "start": { - "line": 17, - "column": 13 - }, - "end": { - "line": 17, - "column": 44 - } - } - } - ], - "kcl_version": "0.2.48" -} diff --git a/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--edit-with-ai-example-snapshots--change-colour.snap.json b/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--edit-with-ai-example-snapshots--change-colour.snap.json deleted file mode 100644 index 48ab1dfb6bc..00000000000 --- a/e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--edit-with-ai-example-snapshots--change-colour.snap.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "original_source_code": "sketch001 = startSketchOn(XZ)\nprofile001 = startProfileAt([57.81, 250.51], sketch001)\n |> line(end = [121.13, 56.63], tag = $seg02)\n |> line(end = [83.37, -34.61], tag = $seg01)\n |> line(end = [19.66, -116.4])\n |> line(end = [-221.8, -41.69])\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude001 = extrude(profile001, length = 200)\nsketch002 = startSketchOn(XZ)\n |> startProfileAt([-73.64, -42.89], %)\n |> xLine(length = 173.71)\n |> line(end = [-22.12, -94.4])\n |> xLine(length = -156.98)\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude002 = extrude(sketch002, length = 50)\nsketch003 = startSketchOn(XY)\n |> startProfileAt([52.92, 157.81], %)\n |> angledLine([0, 176.4], %, $rectangleSegmentA001)\n |> angledLine([\n segAng(rectangleSegmentA001) - 90,\n 53.4\n ], %, $rectangleSegmentB001)\n |> angledLine([\n segAng(rectangleSegmentA001),\n -segLen(rectangleSegmentA001)\n ], %, $rectangleSegmentC001)\n |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n |> close()\nextrude003 = extrude(sketch003, length = 20)\n", - "prompt": "make this neon green please, use #39FF14", - "source_ranges": [ - { - "prompt": "The users main selection is the end cap of a general-sweep (that is an extrusion, revolve, sweep or loft).\nThe source range most likely refers to \"startProfileAt\" simply because this is the start of the profile that was swept.\nIf you need to operate on this cap, for example for sketching on the face, you can use the special string END i.e. `startSketchOn(someSweepVariable, END)`\nWhen they made this selection they main have intended this surface directly or meant something more general like the sweep body.\nSee later source ranges for more context.", - "range": { - "start": { - "line": 11, - "column": 5 - }, - "end": { - "line": 11, - "column": 40 - } - } - }, - { - "prompt": "This is the sweep's source range from the user's main selection of the end cap.", - "range": { - "start": { - "line": 17, - "column": 13 - }, - "end": { - "line": 17, - "column": 44 - } - } - } - ], - "kcl_version": "0.2.57" -} \ No newline at end of file