From 02e5a5de0b6929af64607607ac2c7947e9d6094a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 16 Jul 2026 02:09:44 +0700 Subject: [PATCH] feat(headless): render element locators before lookup Render locator arguments thru the action renderer for every element lookup mode (selector, regex, XPath, JavaScript, and search locators). Keep rendered values local and reuse selectors instead of rendering them twice. Render `waitvisible` locators on the owning page so Interactsh URLs remain attached to the request. Use a local URL source in tests to avoid network access. Closes #7542 Signed-off-by: Dwi Siswanto --- pkg/protocols/headless/engine/instance.go | 3 +- pkg/protocols/headless/engine/page_actions.go | 89 ++++++++++----- .../headless/engine/page_actions_test.go | 107 +++++++++++++++--- 3 files changed, 155 insertions(+), 44 deletions(-) diff --git a/pkg/protocols/headless/engine/instance.go b/pkg/protocols/headless/engine/instance.go index 7646bfa01c..93e1445f8d 100644 --- a/pkg/protocols/headless/engine/instance.go +++ b/pkg/protocols/headless/engine/instance.go @@ -8,6 +8,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/utils" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render" ) // Instance is an isolated browser instance opened for doing operations with it. @@ -16,7 +17,7 @@ type Instance struct { engine *rod.Browser // redundant due to dependency cycle - interactsh *interactsh.Client + interactsh render.URLSource requestLog map[string]string // contains actual request that was sent } diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go index de96a6d343..472b6db431 100644 --- a/pkg/protocols/headless/engine/page_actions.go +++ b/pkg/protocols/headless/engine/page_actions.go @@ -187,9 +187,11 @@ func (p *Page) WaitVisible(act *Action, out ActionData) error { return errors.Wrap(err, "Wrong polling time given") } - element, _ := p.Sleeper(pollTime, timeout). - Timeout(timeout). - pageElementBy(act.Data) + lookupPage := p.Sleeper(pollTime, timeout).Timeout(timeout) + element, _, err := p.pageElementBy(lookupPage.Page(), act) + if err != nil { + return errors.Wrap(err, errElementDidNotAppear) + } if element != nil { if err := element.WaitVisible(); err != nil { @@ -461,7 +463,7 @@ func (p *Page) RunScript(act *Action, out ActionData) error { // ClickElement executes click actions for an element. func (p *Page) ClickElement(act *Action, out ActionData) error { - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -486,7 +488,7 @@ func (p *Page) KeyboardAction(act *Action, out ActionData) error { // RightClickElement executes right click actions for an element. func (p *Page) RightClickElement(act *Action, out ActionData) error { - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -621,7 +623,7 @@ func (p *Page) InputElement(act *Action, out ActionData) error { if value == "" { return errinvalidArguments } - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -643,7 +645,7 @@ func (p *Page) TimeInputElement(act *Action, out ActionData) error { if value == "" { return errinvalidArguments } - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -669,7 +671,7 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error { if value == "" { return errinvalidArguments } - element, err := p.pageElementBy(act.Data) + element, selector, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -688,12 +690,15 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error { selectedBool = true } - selector, err := p.getActionArg(act, "selector") - if err != nil { - return err + if selector == nil { + resolved, err := p.getActionArg(act, "selector") + if err != nil { + return err + } + selector = &resolved } - if err := element.Select([]string{value}, selectedBool, selectorBy(selector)); err != nil { + if err := element.Select([]string{value}, selectedBool, selectorBy(*selector)); err != nil { return errors.Wrap(err, "could not select input") } @@ -744,7 +749,7 @@ func (p *Page) WaitStable(act *Action, out ActionData) error { // GetResource gets a resource from an element from page. func (p *Page) GetResource(act *Action, out ActionData) error { - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -760,7 +765,7 @@ func (p *Page) GetResource(act *Action, out ActionData) error { // FilesInput acts with a file input element on page func (p *Page) FilesInput(act *Action, out ActionData) error { - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -784,7 +789,7 @@ func (p *Page) FilesInput(act *Action, out ActionData) error { // ExtractElement extracts from an element on the page. func (p *Page) ExtractElement(act *Action, out ActionData) error { - element, err := p.pageElementBy(act.Data) + element, _, err := p.pageElementBy(p.page, act) if err != nil { return errors.Wrap(err, errCouldNotGetElement) } @@ -910,32 +915,58 @@ func (p *Page) HandleDialog(act *Action, out ActionData) error { // // Supported values for by: r -> selector & regex, x -> xpath, js -> eval js, // search => query, default ("") => selector. -func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) { - by, ok := data["by"] - if !ok { - by = "" +func (p *Page) pageElementBy(page *rod.Page, action *Action) (*rod.Element, *string, error) { + by, err := p.getActionArg(action, "by") + if err != nil { + return nil, nil, err } - page := p.page switch by { case "r", "regex": - return page.ElementR(data["selector"], data["regex"]) + selector, err := p.getActionArg(action, "selector") + if err != nil { + return nil, nil, err + } + regex, err := p.getActionArg(action, "regex") + if err != nil { + return nil, nil, err + } + element, err := page.ElementR(selector, regex) + return element, &selector, err case "x", "xpath": - return page.ElementX(data["xpath"]) + xpath, err := p.getActionArg(action, "xpath") + if err != nil { + return nil, nil, err + } + element, err := page.ElementX(xpath) + return element, nil, err case "js": - return page.ElementByJS(&rod.EvalOptions{JS: data["js"]}) + js, err := p.getActionArg(action, "js") + if err != nil { + return nil, nil, err + } + element, err := page.ElementByJS(&rod.EvalOptions{JS: js}) + return element, nil, err case "search": - elms, err := page.Search(data["query"]) + query, err := p.getActionArg(action, "query") if err != nil { - return nil, err + return nil, nil, err + } + elms, err := page.Search(query) + if err != nil { + return nil, nil, err } - if elms.First != nil { - return elms.First, nil + return elms.First, nil, nil } - return nil, errors.New("no such element") + return nil, nil, errors.New("no such element") default: - return page.Element(data["selector"]) + selector, err := p.getActionArg(action, "selector") + if err != nil { + return nil, nil, err + } + element, err := page.Element(selector) + return element, &selector, err } } diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index 1e01d2e384..0fd7708520 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -22,13 +22,21 @@ import ( "github.com/projectdiscovery/nuclei/v3/internal/tests/testheadless" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/types" envutil "github.com/projectdiscovery/utils/env" stringsutil "github.com/projectdiscovery/utils/strings" ) +type testInteractshURLSource struct { + calls int +} + +func (s *testInteractshURLSource) NewURLWithData(string) (string, error) { + s.calls++ + return fmt.Sprintf("test-%d.oast.invalid", s.calls), nil +} + func TestGetActionArgTreatsResolvedValuesAsData(t *testing.T) { page := &Page{ instance: &Instance{}, @@ -47,19 +55,10 @@ func TestGetActionArgTreatsResolvedValuesAsData(t *testing.T) { } func TestGetActionArgRendersTemplateInteractshBeforeValidation(t *testing.T) { - client, err := interactsh.New(&interactsh.Options{ - ServerURL: "oast.fun", - CacheSize: 100, - Eviction: 60 * time.Second, - CooldownPeriod: 0, - PollDuration: 5 * time.Second, - DisableHttpFallback: true, - }) - require.NoError(t, err) - defer client.Close() + source := &testInteractshURLSource{} page := &Page{ - instance: &Instance{interactsh: client}, + instance: &Instance{interactsh: source}, mutex: &sync.RWMutex{}, variables: map[string]interface{}{}, } @@ -69,11 +68,91 @@ func TestGetActionArgRendersTemplateInteractshBeforeValidation(t *testing.T) { }}, "value") require.NoError(t, err) + require.Equal(t, 1, source.calls) require.Len(t, page.InteractshURLs, 1) require.NotContains(t, got, "{{interactsh-url}}") require.NotContains(t, got, "%7B%7Binteractsh-url%7D%7D") } +func TestPageElementByRendersLocatorArguments(t *testing.T) { + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + } + response := `` + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, pageErr error, out ActionData) { + require.NoError(t, pageErr) + for key, value := range map[string]interface{}{ + "selector": "button", "text": "target", "xpath": "//button[@id='first']", + "js": "() => document.querySelector('#first')", "query": "target", "mode": "x", + } { + page.variables[key] = value + } + for name, data := range map[string]map[string]string{ + "default": {"selector": "{{selector}}", "xpath": "{{unused}}"}, + "regex": {"by": "r", "selector": "{{selector}}", "regex": "{{text}}"}, + "xpath": {"by": "xpath", "xpath": "{{xpath}}"}, + "javascript": {"by": "js", "js": "{{js}}"}, + "search": {"by": "search", "query": "{{query}}"}, + "rendered mode": {"by": "{{mode}}", "xpath": "{{xpath}}"}, + "static": {"selector": "#first"}, + } { + t.Run(name, func(t *testing.T) { + element, _, err := page.pageElementBy(page.page, &Action{Data: data}) + require.NoError(t, err) + require.Equal(t, "target", element.MustText()) + }) + } + + page.variables["marker"] = "{{runtime}}" + element, _, err := page.pageElementBy(page.page, &Action{Data: map[string]string{ + "by": "xpath", "xpath": "//*[@data-marker='{{marker}}']", + }}) + require.NoError(t, err) + require.Equal(t, "target", element.MustText()) + + action := &Action{Data: map[string]string{"selector": "#{{target}}"}} + page.variables["target"] = "first" + first, _, err := page.pageElementBy(page.page, action) + require.NoError(t, err) + require.Equal(t, "target", first.MustText()) + page.variables["target"] = "second" + second, _, err := page.pageElementBy(page.page, action) + require.NoError(t, err) + require.Equal(t, "second", second.MustText()) + require.Equal(t, "#{{target}}", action.Data["selector"]) + + _, _, err = page.pageElementBy(page.page, &Action{Data: map[string]string{"selector": "{{missing}}"}}) + require.ErrorContains(t, err, "missing") + }) +} + +func TestLocatorInteractshTracking(t *testing.T) { + actions := []*Action{ + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + } + source := &testInteractshURLSource{} + testHeadlessSimpleResponse(t, "", actions, 20*time.Second, func(page *Page, pageErr error, out ActionData) { + require.NoError(t, pageErr) + page.instance.interactsh = source + + err := page.WaitVisible(&Action{Data: map[string]string{ + "selector": "[data-oast='{{interactsh-url}}']", "timeout": "50ms", "pollTime": "10ms", + }}, nil) + require.Error(t, err) + require.Equal(t, 1, source.calls) + require.Len(t, page.InteractshURLs, 1) + + err = page.SelectInputElement(&Action{Data: map[string]string{ + "selector": "select:not([data-oast='{{interactsh-url}}'])", "value": "Test", "selected": "true", + }}, nil) + require.NoError(t, err) + require.Equal(t, 2, source.calls) + require.Len(t, page.InteractshURLs, 2) + }) +} + func TestActionNavigate(t *testing.T) { response := ` @@ -150,7 +229,7 @@ func TestActionClick(t *testing.T) { actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, - {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "{{to_lower('BUTTON')}}"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out ActionData) { @@ -747,7 +826,7 @@ func TestActionWaitVisible(t *testing.T) { actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionWaitVisible}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitVisible}, Data: map[string]string{"by": "x", "xpath": "//button[@id='{{to_lower('TEST')}}']"}}, } t.Run("wait for an element being visible", func(t *testing.T) {