Merge pull request #29 from zerostep-ai/v0.1.5

merge version v0.1.5

examples/playwright-demo/tests/zerostep-regression.spec.ts

@@ -88,4 +88,60 @@ test.describe('JSPaint', () => {
     const foregroundColor = await foregroundColorHandle.jsonValue()
     expect(foregroundColor).toEqual('rgba(0,0,0,1)')
   })
-})
+})
+
+test.describe('Reflect', () => {
+  test('can scroll elements to the bottom', async ({ page, ai }) => {
+    await page.goto('https://reflect.run/docs/')
+    await ai('Scroll the sidebar navigation to the bottom')
+
+    const scrollTop = await page.evaluate(() => {
+      return document.querySelector('#docs-left-nav > div')?.scrollTop
+    })
+
+    console.log('scrollTop', scrollTop)
+
+    expect(scrollTop).toBeTruthy()
+  })
+
+  test('can scroll elements to the top', async ({ page, ai }) => {
+    await page.goto('https://reflect.run/docs/')
+    await ai('Scroll the sidebar navigation to the bottom')
+    await ai('Scroll the sidebar navigation to the top')
+
+    const scrollTop = await page.evaluate(() => {
+      return document.querySelector('#docs-left-nav > div')?.scrollTop
+    })
+
+    console.log('scrollTop', scrollTop)
+
+    expect(scrollTop).toBe(0)
+  })
+
+  test('can scroll elements down', async ({ page, ai }) => {
+    await page.goto('https://reflect.run/docs/')
+    await ai('Scroll the sidebar navigation down')
+
+    const scrollTop = await page.evaluate(() => {
+      return document.querySelector('#docs-left-nav > div')?.scrollTop
+    })
+
+    console.log('scrollTop', scrollTop)
+
+    expect(scrollTop).toBeTruthy()
+  })
+
+  test('can scroll elements up', async ({ page, ai }) => {
+    await page.goto('https://reflect.run/docs/')
+    await ai('Scroll the sidebar navigation down')
+    await ai('Scroll the sidebar navigation up')
+
+    const scrollTop = await page.evaluate(() => {
+      return document.querySelector('#docs-left-nav > div')?.scrollTop
+    })
+
+    console.log('scrollTop', scrollTop)
+
+    expect(scrollTop).toBe(0)
+  })
+})

examples/playwright-demo/zerostep.config.json

@@ -1,3 +1,4 @@
 {
-  "TOKEN": "<drop token here if not using an environment variable>"
+  "TOKEN": "<drop token here if not using an environment variable>",
+  "LOGS_ENABLED": true
 }

packages/playwright/CHANGELOG.md

@@ -4,6 +4,14 @@ Documentation on notable changes to the package and release notes
 
 ---
 
+## [0.1.5](https://www.npmjs.com/package/@zerostep/playwright/v/0.1.5) - 12-08-2023
+
+**What's New**
+
+- Can execute scroll commands against specific elements, including those in iframes
+
+---
+
 ## [0.1.4](https://www.npmjs.com/package/@zerostep/playwright/v/0.1.4) - 12-01-2023
 
 **What's New**

packages/playwright/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zerostep/playwright",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Supercharge your Playwright tests with AI",
   "author": "zerostep",
   "license": "MIT",
@@ -15,6 +15,7 @@
   ],
   "scripts": {
     "build": "tsc",
+    "watch": "tsc --watch",
     "prepublishOnly": "mv README.md tmp.README.md && cp ../../README.md README.md",
     "postpublish": "mv tmp.README.md README.md",
     "release": "npm run build && npm publish --access public"

packages/playwright/src/cdp.ts

@@ -1,4 +1,4 @@
-import { type Page } from './types.js'
+import type { Page, ScrollType } from './types.js'
 import { WEBDRIVER_ELEMENT_KEY } from './config.js'
 
 let cdpSessionByPage = new Map<Page, CDPSession>()
@@ -62,19 +62,57 @@ export const get = async (page: Page, args: { url: string }) => {
   })
 }
 
-export const runFunctionOn = async (page: Page, args: { functionDeclaration: string, objectId: string }) => {
+export const scrollElement = async (page: Page, args: { id: string, target: ScrollType }) => {
+  await runFunctionOn(page, {
+    functionDeclaration: `function() {
+      let element = this
+      let elementHeight = 0
+
+      switch (element.tagName) {
+        case 'BODY':
+        case 'HTML':
+          element = document.scrollingElement || document.body
+          elementHeight = window.visualViewport?.height ?? 720
+          break
+        default:
+          elementHeight = element.clientHeight ?? 720
+          break
+      }
+
+      const relativeScrollDistance = 0.75 * elementHeight
+
+      switch ("${args.target}") {
+        case 'top':
+          return element.scrollTo({ top: 0 })
+        case 'bottom':
+          return element.scrollTo({ top: element.scrollHeight })
+        case 'up':
+          return element.scrollBy({ top: -relativeScrollDistance })
+        case 'down':
+          return element.scrollBy({ top: relativeScrollDistance })
+        default:
+          throw Error('Unsupported scroll target ${args.target}')
+      }
+    }`,
+    backendNodeId: parseInt(args.id),
+  })
+}
+
+export const runFunctionOn = async (page: Page, args: { functionDeclaration: string, backendNodeId: number }) => {
   const cdpSession = await getCDPSession(page)
+  const { object: { objectId } } = await cdpSession.send('DOM.resolveNode', { backendNodeId: args.backendNodeId })
   await cdpSession.send('Runtime.callFunctionOn', {
     functionDeclaration: args.functionDeclaration,
-    objectId: args.objectId,
+    objectId,
   })
 }
 
 export const getDOMSnapshot = async (page: Page) => {
   const cdpSession = await getCDPSession(page)
   const returnValue = await cdpSession.send('DOMSnapshot.captureSnapshot', {
-    computedStyles: ['background-color', 'visibility', 'opacity', 'z-index'],
-    includePaintOrder: true
+    computedStyles: ['background-color', 'visibility', 'opacity', 'z-index', 'overflow'],
+    includePaintOrder: true,
+    includeDOMRects: true,
   })
   return returnValue
 }
@@ -86,11 +124,9 @@ export const getLayoutMetrics = async (page: Page) => {
 }
 
 export const clearElement = async (page: Page, args: { id: string }) => {
-  const cdpSession = await getCDPSession(page)
-  const { object: { objectId } } = await cdpSession.send('DOM.resolveNode', { backendNodeId: parseInt(args.id) })
-  await runFunctionOn(page, {
+  return await runFunctionOn(page, {
     functionDeclaration: `function() {this.value=''}`,
-    objectId: objectId!,
+    backendNodeId: parseInt(args.id),
   })
 }
 

packages/playwright/src/index.ts

@@ -4,13 +4,13 @@ import * as playwright from './playwright.js'
 import * as webSocket from './webSocket.js'
 import * as meta from './meta.js'
 import { PACKAGE_NAME, MAX_TASK_CHARS, TOKEN } from './config.js'
-import { type APIPage, type APITestType, type Page, type TestType } from './types.js'
-import {
-  type CommandRequestZeroStepMessage,
-  type CommandResponseZeroStepMessage,
-  type TaskCompleteZeroStepMessage,
-  type TaskStartZeroStepMessage,
-  type StepOptions
+import type { APIPage, APITestType, Page, TestType, ScrollType } from './types.js'
+import type {
+  CommandRequestZeroStepMessage,
+  CommandResponseZeroStepMessage,
+  TaskCompleteZeroStepMessage,
+  TaskStartZeroStepMessage,
+  StepOptions
 } from './webSocket.js'
 
 /**
@@ -212,6 +212,8 @@ const executeCommand = async (page: Page, command: CommandRequestZeroStepMessage
       return await playwright.clickAndInputCDPElement(page, command.arguments as { id: string, value: string })
     case 'hoverElement':
       return await playwright.hoverCDPElement(page, command.arguments as { id: string })
+    case 'scrollElement':
+      return await cdp.scrollElement(page, command.arguments as { id: string, target: ScrollType })
 
     // Actions using Location
     case 'clickLocation':
@@ -233,7 +235,7 @@ const executeCommand = async (page: Page, command: CommandRequestZeroStepMessage
 
     // Actions using Script
     case 'scrollPage':
-      return await playwright.scrollPageScript(page, command.arguments as { target: playwright.ScrollType })
+      return await playwright.scrollPageScript(page, command.arguments as { target: ScrollType })
 
     default:
       throw Error(`Unsupported command ${command.name}`)

packages/playwright/src/playwright.ts

@@ -1,4 +1,4 @@
-import { type Page, type ElementHandle, type Frame } from './types.js'
+import type { Page, ElementHandle, Frame, ScrollType } from './types.js'
 import * as cdp from './cdp.js'
 
 // Actions using CDP Element
@@ -17,6 +17,11 @@ export const clickAndInputCDPElement = async (page: Page, args: { id: string, va
   await clickAndInputLocation(page, { x: centerX, y: centerY, value: args.value })
 }
 
+export const scrollCDPElement = async (page: Page, args: { id: string, target: ScrollType }) => {
+  const element = await cdpElementToPlaywrightHandle(page, { backendNodeId: parseInt(args.id) })
+  await scrollElementScript(page, { element, target: args.target })
+}
+
 // Actions using Location
 export const hoverLocation = async (page: Page, args: { x: number, y: number }) => {
   const { element, tagName, isCustomElement } = await getElementAtLocation(page, args)
@@ -127,6 +132,28 @@ export const scrollPageScript = async (page: Page, args: { target: ScrollType })
   }, args)
 }
 
+export const scrollElementScript = async (page: Page, args: { element: ElementHandle<Element>, target: ScrollType }) => {
+  await args.element.evaluate((element, evalArgs) => {
+    // The element height should be defined, but if it somehow isn't pick a reasonable default
+    const elementHeight = element.clientHeight ?? 720
+    // For relative scrolls, attempt to scroll by 75% of the element height
+    const relativeScrollDistance = 0.75 * elementHeight
+
+    switch (evalArgs.target) {
+      case 'top':
+        return element.scrollTo({ top: 0 })
+      case 'bottom':
+        return element.scrollTo({ top: element.scrollHeight })
+      case 'up':
+        return element.scrollBy({ top: -relativeScrollDistance })
+      case 'down':
+        return element.scrollBy({ top: relativeScrollDistance })
+      default:
+        throw Error(`Unsupported scroll target ${evalArgs.target}`)
+    }
+  }, args)
+}
+
 // Meta
 export const getViewportMetadata = async (page: Page) => {
   const metadata = await page.evaluate(() => {
@@ -156,6 +183,29 @@ export const getSnapshot = async (page: Page) => {
   return { dom, screenshot, viewportWidth, viewportHeight, pixelRatio, layoutMetrics }
 }
 
+export const cdpElementToPlaywrightHandle = async (page: Page, args: { backendNodeId: number }) => {
+  await storeCDPElement(page, args)
+  const element = await getStoredCDPElementRef(page)
+  await clearStoredCDPElementRef(page)
+  return element
+}
+
+export const storeCDPElement = async (page: Page, args: { backendNodeId: number }) => {
+  await cdp.runFunctionOn(page, {
+    functionDeclaration: `function() { window.$$ZEROSTEP_TEMP_NODE = this }`,
+    backendNodeId: args.backendNodeId
+  })
+}
+
+export const getStoredCDPElementRef = async (page: Page) => {
+  const handle = await page.evaluateHandle(() => window['$$ZEROSTEP_TEMP_NODE' as any] as unknown as Element)
+  return handle.asElement()
+}
+
+export const clearStoredCDPElementRef = async (page: Page) => {
+  return await page.evaluateHandle(() => delete window['$$ZEROSTEP_TEMP_NODE' as any])
+}
+
 export const getElementAtLocation = async (
   context: Page | Frame | ElementHandle<ShadowRoot>,
   args: { x: number, y: number, isShadowRoot?: boolean }
@@ -205,9 +255,3 @@ export const getElementAtLocation = async (
     isCustomElement,
   }
 }
-
-export type ScrollType =
-  | 'up'
-  | 'down'
-  | 'bottom'
-  | 'top'

packages/playwright/src/types.ts

@@ -9,3 +9,10 @@ export { type Page, type ElementHandle, type Frame } from '@playwright/test'
 // are only used for typesafety.
 export type APIPage = Pick<Page, 'mouse' | 'keyboard'>
 export type APITestType = Pick<TestType<any, any>, 'step'>
+
+// Step-specific types
+export type ScrollType =
+  | 'up'
+  | 'down'
+  | 'bottom'
+  | 'top'