diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index c9dc3385b..57be015ed 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -541,8 +541,7 @@ export class WorkflowGenerator { * @returns {Promise} */ private generateSelector = async (page: Page, coordinates: Coordinates, action: ActionType) => { - const elementInfo = await getElementInformation(page, coordinates); - + const elementInfo = await getElementInformation(page, coordinates, this.listSelector); const selectorBasedOnCustomAction = (this.getList === true) ? await getNonUniqueSelectors(page, coordinates) : await getSelectors(page, coordinates); @@ -570,16 +569,14 @@ export class WorkflowGenerator { * @returns {Promise} */ public generateDataForHighlighter = async (page: Page, coordinates: Coordinates) => { - const rect = await getRect(page, coordinates); + const rect = await getRect(page, coordinates, this.listSelector); const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click); - const elementInfo = await getElementInformation(page, coordinates); + const elementInfo = await getElementInformation(page, coordinates, this.listSelector); if (rect) { if (this.getList === true) { if (this.listSelector !== '') { const childSelectors = await getChildSelectors(page, this.listSelector || ''); this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo, childSelectors }) - console.log(`Child Selectors: ${childSelectors}`) - console.log(`Parent Selector: ${this.listSelector}`) } else { this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo }); } diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 917ac561c..891c0e3ba 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -1,121 +1,156 @@ import { Page } from "playwright"; -import { Action, ActionType, Coordinates, TagName } from "../types"; +import { Coordinates } from "../types"; import { WhereWhatPair, WorkflowFile } from "maxun-core"; import logger from "../logger"; -import { getBestSelectorForAction } from "./utils"; - -/*TODO: -1. Handle TS errors (here we definetly know better) -2. Add pending function descriptions + thought process (esp. selector generation) -*/ type Workflow = WorkflowFile["workflow"]; /** - * Returns a {@link Rectangle} object representing - * the coordinates, width, height and corner points of the element. - * If an element is not found, returns null. + * Checks the basic info about an element and returns a {@link BaseActionInfo} object. + * If the element is not found, returns undefined. * @param page The page instance. * @param coordinates Coordinates of an element. * @category WorkflowManagement-Selectors - * @returns {Promise} + * @returns {Promise} */ export const getElementInformation = async ( page: Page, - coordinates: Coordinates + coordinates: Coordinates, + listSelector: string, ) => { try { - const elementInfo = await page.evaluate( - async ({ x, y }) => { - const originalEl = document.elementFromPoint(x, y) as HTMLElement; - if (originalEl) { - let element = originalEl; - - // if (originalEl.tagName === 'A') { - // element = originalEl; - // } else if (originalEl.parentElement?.tagName === 'A') { - // element = originalEl.parentElement; - // } else { - // Generic parent finding logic based on visual containment - const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', - 'ADDRESS', 'BLOCKQUOTE', 'DETAILS', 'DIALOG', 'FIGURE', 'FIGCAPTION', 'MAIN', 'MARK', 'SUMMARY', 'TIME', - 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TR', 'TH', 'TD', 'CAPTION', 'COLGROUP', 'COL', 'FORM', 'FIELDSET', - 'LEGEND', 'LABEL', 'INPUT', 'BUTTON', 'SELECT', 'DATALIST', 'OPTGROUP', 'OPTION', 'TEXTAREA', 'OUTPUT', - 'PROGRESS', 'METER', 'DETAILS', 'SUMMARY', 'MENU', 'MENUITEM', 'MENUITEM', 'APPLET', 'EMBED', 'OBJECT', - 'PARAM', 'VIDEO', 'AUDIO', 'SOURCE', 'TRACK', 'CANVAS', 'MAP', 'AREA', 'SVG', 'IFRAME', 'FRAME', 'FRAMESET', - 'LI', 'UL', 'OL', 'DL', 'DT', 'DD', 'HR', 'P', 'PRE', 'LISTING', 'PLAINTEXT', 'A' - ]; - while (element.parentElement) { - const parentRect = element.parentElement.getBoundingClientRect(); - const childRect = element.getBoundingClientRect(); - - if (!containerTags.includes(element.parentElement.tagName)) { - break; + if (listSelector !== '') { + const elementInfo = await page.evaluate( + async ({ x, y }) => { + const el = document.elementFromPoint(x, y) as HTMLElement; + if (el) { + const { parentElement } = el; + const element = parentElement?.tagName === 'A' ? parentElement : el; + let info: { + tagName: string; + hasOnlyText?: boolean; + innerText?: string; + url?: string; + imageUrl?: string; + attributes?: Record; + innerHTML?: string; + outerHTML?: string; + } = { + tagName: element?.tagName ?? '', + }; + if (element) { + info.attributes = Array.from(element.attributes).reduce( + (acc, attr) => { + acc[attr.name] = attr.value; + return acc; + }, + {} as Record + ); } + // Gather specific information based on the tag + if (element?.tagName === 'A') { + info.url = (element as HTMLAnchorElement).href; + info.innerText = element.innerText ?? ''; + } else if (element?.tagName === 'IMG') { + info.imageUrl = (element as HTMLImageElement).src; + } else { + info.hasOnlyText = element?.children?.length === 0 && + element?.innerText?.length > 0; + info.innerText = element?.innerText ?? ''; + } + info.innerHTML = element.innerHTML; + info.outerHTML = element.outerHTML; + return info; + } + return null; + }, + { x: coordinates.x, y: coordinates.y }, + ); + return elementInfo; + } else { + const elementInfo = await page.evaluate( + async ({ x, y }) => { + const originalEl = document.elementFromPoint(x, y) as HTMLElement; + if (originalEl) { + let element = originalEl; + + const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', + 'ADDRESS', 'BLOCKQUOTE', 'DETAILS', 'DIALOG', 'FIGURE', 'FIGCAPTION', 'MAIN', 'MARK', 'SUMMARY', 'TIME', + 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TR', 'TH', 'TD', 'CAPTION', 'COLGROUP', 'COL', 'FORM', 'FIELDSET', + 'LEGEND', 'LABEL', 'INPUT', 'BUTTON', 'SELECT', 'DATALIST', 'OPTGROUP', 'OPTION', 'TEXTAREA', 'OUTPUT', + 'PROGRESS', 'METER', 'DETAILS', 'SUMMARY', 'MENU', 'MENUITEM', 'MENUITEM', 'APPLET', 'EMBED', 'OBJECT', + 'PARAM', 'VIDEO', 'AUDIO', 'SOURCE', 'TRACK', 'CANVAS', 'MAP', 'AREA', 'SVG', 'IFRAME', 'FRAME', 'FRAMESET', + 'LI', 'UL', 'OL', 'DL', 'DT', 'DD', 'HR', 'P', 'PRE', 'LISTING', 'PLAINTEXT', 'A' + ]; + while (element.parentElement) { + const parentRect = element.parentElement.getBoundingClientRect(); + const childRect = element.getBoundingClientRect(); + + if (!containerTags.includes(element.parentElement.tagName)) { + break; + } - // Check if parent visually contains the child - const fullyContained = - parentRect.left <= childRect.left && - parentRect.right >= childRect.right && - parentRect.top <= childRect.top && - parentRect.bottom >= childRect.bottom; + const fullyContained = + parentRect.left <= childRect.left && + parentRect.right >= childRect.right && + parentRect.top <= childRect.top && + parentRect.bottom >= childRect.bottom; - // Additional checks for more comprehensive containment - const significantOverlap = - (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + const significantOverlap = + (childRect.width * childRect.height) / + (parentRect.width * parentRect.height) > 0.5; - if (fullyContained && significantOverlap) { - element = element.parentElement; - } else { - break; - // } - } } - - let info: { - tagName: string; - hasOnlyText?: boolean; - innerText?: string; - url?: string; - imageUrl?: string; - attributes?: Record; - innerHTML?: string; - outerHTML?: string; - } = { - tagName: element?.tagName ?? '', - }; + if (fullyContained && significantOverlap) { + element = element.parentElement; + } else { + break; + } + } - if (element) { - info.attributes = Array.from(element.attributes).reduce( - (acc, attr) => { - acc[attr.name] = attr.value; - return acc; - }, - {} as Record - ); - } + let info: { + tagName: string; + hasOnlyText?: boolean; + innerText?: string; + url?: string; + imageUrl?: string; + attributes?: Record; + innerHTML?: string; + outerHTML?: string; + } = { + tagName: element?.tagName ?? '', + }; - // Existing tag-specific logic - if (element?.tagName === 'A') { - info.url = (element as HTMLAnchorElement).href; - info.innerText = element.innerText ?? ''; - } else if (element?.tagName === 'IMG') { - info.imageUrl = (element as HTMLImageElement).src; - } else { - info.hasOnlyText = element?.children?.length === 0 && - element?.innerText?.length > 0; - info.innerText = element?.innerText ?? ''; - } + if (element) { + info.attributes = Array.from(element.attributes).reduce( + (acc, attr) => { + acc[attr.name] = attr.value; + return acc; + }, + {} as Record + ); + } - info.innerHTML = element.innerHTML; - info.outerHTML = element.outerHTML; - return info; - } - return null; - }, - { x: coordinates.x, y: coordinates.y }, - ); - return elementInfo; + if (element?.tagName === 'A') { + info.url = (element as HTMLAnchorElement).href; + info.innerText = element.innerText ?? ''; + } else if (element?.tagName === 'IMG') { + info.imageUrl = (element as HTMLImageElement).src; + } else { + info.hasOnlyText = element?.children?.length === 0 && + element?.innerText?.length > 0; + info.innerText = element?.innerText ?? ''; + } + + info.innerHTML = element.innerHTML; + info.outerHTML = element.outerHTML; + return info; + } + return null; + }, + { x: coordinates.x, y: coordinates.y }, + ); + return elementInfo; + } } catch (error) { const { message, stack } = error as Error; console.error('Error while retrieving selector:', message); @@ -123,79 +158,110 @@ export const getElementInformation = async ( } }; -export const getRect = async (page: Page, coordinates: Coordinates) => { +/** + * Returns a {@link Rectangle} object representing + * the coordinates, width, height and corner points of the element. + * If an element is not found, returns null. + * @param page The page instance. + * @param coordinates Coordinates of an element. + * @category WorkflowManagement-Selectors + * @returns {Promise} + */ +export const getRect = async (page: Page, coordinates: Coordinates, listSelector: string) => { try { - const rect = await page.evaluate( - async ({ x, y }) => { - const originalEl = document.elementFromPoint(x, y) as HTMLElement; - if (originalEl) { - let element = originalEl; - - // if (originalEl.tagName === 'A') { - // element = originalEl; - // } else if (originalEl.parentElement?.tagName === 'A') { - // element = originalEl.parentElement; - // } else { - const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', - 'ADDRESS', 'BLOCKQUOTE', 'DETAILS', 'DIALOG', 'FIGURE', 'FIGCAPTION', 'MAIN', 'MARK', 'SUMMARY', 'TIME', - 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TR', 'TH', 'TD', 'CAPTION', 'COLGROUP', 'COL', 'FORM', 'FIELDSET', - 'LEGEND', 'LABEL', 'INPUT', 'BUTTON', 'SELECT', 'DATALIST', 'OPTGROUP', 'OPTION', 'TEXTAREA', 'OUTPUT', - 'PROGRESS', 'METER', 'DETAILS', 'SUMMARY', 'MENU', 'MENUITEM', 'MENUITEM', 'APPLET', 'EMBED', 'OBJECT', - 'PARAM', 'VIDEO', 'AUDIO', 'SOURCE', 'TRACK', 'CANVAS', 'MAP', 'AREA', 'SVG', 'IFRAME', 'FRAME', 'FRAMESET', - 'LI', 'UL', 'OL', 'DL', 'DT', 'DD', 'HR', 'P', 'PRE', 'LISTING', 'PLAINTEXT', 'A' - ]; - while (element.parentElement) { - const parentRect = element.parentElement.getBoundingClientRect(); - const childRect = element.getBoundingClientRect(); - - if (!containerTags.includes(element.parentElement.tagName)) { - break; + if (listSelector !== '') { + const rect = await page.evaluate( + async ({ x, y }) => { + const el = document.elementFromPoint(x, y) as HTMLElement; + if (el) { + const { parentElement } = el; + // Match the logic in recorder.ts for link clicks + const element = parentElement?.tagName === 'A' ? parentElement : el; + const rectangle = element?.getBoundingClientRect(); + if (rectangle) { + return { + x: rectangle.x, + y: rectangle.y, + width: rectangle.width, + height: rectangle.height, + top: rectangle.top, + right: rectangle.right, + bottom: rectangle.bottom, + left: rectangle.left, + }; } - + } + }, + { x: coordinates.x, y: coordinates.y }, + ); + return rect; + } else { + const rect = await page.evaluate( + async ({ x, y }) => { + const originalEl = document.elementFromPoint(x, y) as HTMLElement; + if (originalEl) { + let element = originalEl; + + const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', + 'ADDRESS', 'BLOCKQUOTE', 'DETAILS', 'DIALOG', 'FIGURE', 'FIGCAPTION', 'MAIN', 'MARK', 'SUMMARY', 'TIME', + 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TR', 'TH', 'TD', 'CAPTION', 'COLGROUP', 'COL', 'FORM', 'FIELDSET', + 'LEGEND', 'LABEL', 'INPUT', 'BUTTON', 'SELECT', 'DATALIST', 'OPTGROUP', 'OPTION', 'TEXTAREA', 'OUTPUT', + 'PROGRESS', 'METER', 'DETAILS', 'SUMMARY', 'MENU', 'MENUITEM', 'MENUITEM', 'APPLET', 'EMBED', 'OBJECT', + 'PARAM', 'VIDEO', 'AUDIO', 'SOURCE', 'TRACK', 'CANVAS', 'MAP', 'AREA', 'SVG', 'IFRAME', 'FRAME', 'FRAMESET', + 'LI', 'UL', 'OL', 'DL', 'DT', 'DD', 'HR', 'P', 'PRE', 'LISTING', 'PLAINTEXT', 'A' + ]; + while (element.parentElement) { + const parentRect = element.parentElement.getBoundingClientRect(); + const childRect = element.getBoundingClientRect(); + + if (!containerTags.includes(element.parentElement.tagName)) { + break; + } - const fullyContained = - parentRect.left <= childRect.left && - parentRect.right >= childRect.right && - parentRect.top <= childRect.top && - parentRect.bottom >= childRect.bottom; + const fullyContained = + parentRect.left <= childRect.left && + parentRect.right >= childRect.right && + parentRect.top <= childRect.top && + parentRect.bottom >= childRect.bottom; - const significantOverlap = - (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + const significantOverlap = + (childRect.width * childRect.height) / + (parentRect.width * parentRect.height) > 0.5; - if (fullyContained && significantOverlap) { - element = element.parentElement; - } else { - break; - // } - }} - - //element = element?.parentElement?.tagName === 'A' ? element?.parentElement : element; - const rectangle = element?.getBoundingClientRect(); - - if (rectangle) { - return { - x: rectangle.x, - y: rectangle.y, - width: rectangle.width, - height: rectangle.height, - top: rectangle.top, - right: rectangle.right, - bottom: rectangle.bottom, - left: rectangle.left, - }; + if (fullyContained && significantOverlap) { + element = element.parentElement; + } else { + break; + } + } + + const rectangle = element?.getBoundingClientRect(); + + if (rectangle) { + return { + x: rectangle.x, + y: rectangle.y, + width: rectangle.width, + height: rectangle.height, + top: rectangle.top, + right: rectangle.right, + bottom: rectangle.bottom, + left: rectangle.left, + }; + } } - } - }, - { x: coordinates.x, y: coordinates.y }, - ); - return rect; + return null; + }, + { x: coordinates.x, y: coordinates.y }, + ); + return rect; + } } catch (error) { const { message, stack } = error as Error; logger.log('error', `Error while retrieving selector: ${message}`); logger.log('error', `Stack: ${stack}`); } -} +}; /** @@ -848,7 +914,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates let element = originalEl; - const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', + const containerTags = ['DIV', 'SECTION', 'ARTICLE', 'MAIN', 'HEADER', 'FOOTER', 'NAV', 'ASIDE', 'ADDRESS', 'BLOCKQUOTE', 'DETAILS', 'DIALOG', 'FIGURE', 'FIGCAPTION', 'MAIN', 'MARK', 'SUMMARY', 'TIME', 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TR', 'TH', 'TD', 'CAPTION', 'COLGROUP', 'COL', 'FORM', 'FIELDSET', 'LEGEND', 'LABEL', 'INPUT', 'BUTTON', 'SELECT', 'DATALIST', 'OPTGROUP', 'OPTION', 'TEXTAREA', 'OUTPUT', @@ -856,7 +922,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates 'PARAM', 'VIDEO', 'AUDIO', 'SOURCE', 'TRACK', 'CANVAS', 'MAP', 'AREA', 'SVG', 'IFRAME', 'FRAME', 'FRAMESET', 'LI', 'UL', 'OL', 'DL', 'DT', 'DD', 'HR', 'P', 'PRE', 'LISTING', 'PLAINTEXT', 'A' ]; - + while (element.parentElement) { const parentRect = element.parentElement.getBoundingClientRect(); const childRect = element.getBoundingClientRect(); @@ -865,22 +931,22 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } - const fullyContained = + const fullyContained = parentRect.left <= childRect.left && parentRect.right >= childRect.right && parentRect.top <= childRect.top && parentRect.bottom >= childRect.bottom; - const significantOverlap = - (childRect.width * childRect.height) / + const significantOverlap = + (childRect.width * childRect.height) / (parentRect.width * parentRect.height) > 0.5; if (fullyContained && significantOverlap) { element = element.parentElement; } else { break; + } } - } const generalSelector = getSelectorPath(element); return {