From 7c42305ef56a28b52ec4bb6fa2bfeea571d64508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 26 Mar 2024 15:42:46 +0300 Subject: [PATCH 001/183] fix lib log config issue --- internal/runner/options.go | 17 ----------------- internal/runner/runner.go | 1 - pkg/types/crawler_options.go | 19 +++++++++++++++++++ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index d6380f55..0c7fd0fb 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -9,12 +9,10 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" - "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/types" "github.com/projectdiscovery/katana/pkg/utils" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" - logutil "github.com/projectdiscovery/utils/log" "gopkg.in/yaml.v3" ) @@ -110,21 +108,6 @@ func normalizeInput(value string) string { return strings.TrimSpace(value) } -// configureOutput configures the output logging levels to be displayed on the screen -func configureOutput(options *types.Options) { - if options.Silent { - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - } else if options.Verbose { - gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) - } else if options.Debug { - gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - } else { - gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) - } - - logutil.DisableDefaultLogger() -} - func initExampleFormFillConfig() error { homedir, err := os.UserHomeDir() if err != nil { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 26a0f6e4..03d9118e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -57,7 +57,6 @@ func New(options *types.Options) (*Runner, error) { options.URLs = mapsutil.GetKeys(runnerState.InFlightUrls.GetAll()) } - configureOutput(options) showBanner() if options.Version { diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index e0e0212b..b03b53de 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -6,12 +6,15 @@ import ( "time" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/output" "github.com/projectdiscovery/katana/pkg/utils/extensions" "github.com/projectdiscovery/katana/pkg/utils/filters" "github.com/projectdiscovery/katana/pkg/utils/scope" "github.com/projectdiscovery/ratelimit" errorutil "github.com/projectdiscovery/utils/errors" + logutil "github.com/projectdiscovery/utils/log" urlutil "github.com/projectdiscovery/utils/url" wappalyzer "github.com/projectdiscovery/wappalyzergo" ) @@ -39,6 +42,7 @@ type CrawlerOptions struct { // NewCrawlerOptions creates a new crawler options structure // from user specified options. func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { + configureOutput(options) extensionsValidator := extensions.NewValidator(options.ExtensionsMatch, options.ExtensionFilter) dialerOpts := fastdialer.DefaultOptions @@ -147,3 +151,18 @@ func (c *CrawlerOptions) ValidateScope(absURL, rootHostname string) (bool, error } return true, nil } + +// configureOutput configures the output logging levels to be displayed on the screen +func configureOutput(options *Options) { + if options.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } else if options.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) + } else if options.Debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } else { + gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) + } + + logutil.DisableDefaultLogger() +} From 664f8865103a3555bf06c422c10b3a36a472382b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 29 Mar 2024 13:21:29 +0300 Subject: [PATCH 002/183] use process util --- pkg/engine/hybrid/hybrid.go | 56 +++---------------------------------- 1 file changed, 4 insertions(+), 52 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 89392ad3..7133ff06 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -11,10 +11,8 @@ import ( "github.com/projectdiscovery/katana/pkg/engine/common" "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" - stringsutil "github.com/projectdiscovery/utils/strings" + processutil "github.com/projectdiscovery/utils/process" urlutil "github.com/projectdiscovery/utils/url" - ps "github.com/shirou/gopsutil/v3/process" - "go.uber.org/multierr" ) // Crawler is a standard crawler instance @@ -39,7 +37,7 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { } } - previousPIDs := findChromeProcesses() + previousPIDs := processutil.FindProcesses(processutil.IsChromeProcess) var launcherURL string var chromeLauncher *launcher.Launcher @@ -104,7 +102,8 @@ func (c *Crawler) Close() error { return err } } - return c.killChromeProcesses() + processutil.CloseProcesses(processutil.IsChromeProcess, c.previousPIDs) + return nil } // Crawl crawls a URL with the specified options @@ -173,50 +172,3 @@ func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*laun return chromeLauncher, nil } - -// killChromeProcesses any and all new chrome processes started after -// headless process launch. -func (c *Crawler) killChromeProcesses() error { - var errs []error - processes, _ := ps.Processes() - - for _, process := range processes { - // skip non-chrome processes - if !isChromeProcess(process) { - continue - } - - // skip chrome processes that were already running - if _, ok := c.previousPIDs[process.Pid]; ok { - continue - } - - if err := process.Kill(); err != nil { - errs = append(errs, err) - } - } - - return multierr.Combine(errs...) -} - -// findChromeProcesses finds chrome process running on host -func findChromeProcesses() map[int32]struct{} { - processes, _ := ps.Processes() - list := make(map[int32]struct{}) - for _, process := range processes { - if isChromeProcess(process) { - list[process.Pid] = struct{}{} - if ppid, err := process.Ppid(); err == nil { - list[ppid] = struct{}{} - } - } - } - return list -} - -// isChromeProcess checks if a process is chrome/chromium -func isChromeProcess(process *ps.Process) bool { - name, _ := process.Name() - executable, _ := process.Exe() - return stringsutil.ContainsAny(name, "chrome", "chromium") || stringsutil.ContainsAny(executable, "chrome", "chromium") -} From 5ba071437ea96faae8afe06b1d33602c697d250f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 29 Mar 2024 13:22:32 +0300 Subject: [PATCH 003/183] enable leakless --- pkg/engine/hybrid/hybrid.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 7133ff06..bc9e3e5e 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -11,7 +11,6 @@ import ( "github.com/projectdiscovery/katana/pkg/engine/common" "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" - processutil "github.com/projectdiscovery/utils/process" urlutil "github.com/projectdiscovery/utils/url" ) @@ -19,9 +18,14 @@ import ( type Crawler struct { *common.Shared - browser *rod.Browser - previousPIDs map[int32]struct{} // track already running PIDs - tempDir string + browser *rod.Browser + // TODO: Remove the Chrome PID kill code in favor of using Leakless(true). + // This change will be made if there are no complaints about zombie Chrome processes. + // References: + // https://github.com/projectdiscovery/katana/issues/632 + // https://github.com/projectdiscovery/httpx/issues/1425 + // previousPIDs map[int32]struct{} // track already running PIDs + tempDir string } // New returns a new standard crawler instance @@ -37,7 +41,7 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { } } - previousPIDs := processutil.FindProcesses(processutil.IsChromeProcess) + // previousPIDs := processutil.FindProcesses(processutil.IsChromeProcess) var launcherURL string var chromeLauncher *launcher.Launcher @@ -81,10 +85,10 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { } crawler := &Crawler{ - Shared: shared, - browser: browser, - previousPIDs: previousPIDs, - tempDir: dataStore, + Shared: shared, + browser: browser, + // previousPIDs: previousPIDs, + tempDir: dataStore, } return crawler, nil @@ -102,7 +106,7 @@ func (c *Crawler) Close() error { return err } } - processutil.CloseProcesses(processutil.IsChromeProcess, c.previousPIDs) + // processutil.CloseProcesses(processutil.IsChromeProcess, c.previousPIDs) return nil } @@ -125,7 +129,7 @@ func (c *Crawler) Crawl(rootURL string) error { // buildChromeLauncher builds a new chrome launcher instance func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*launcher.Launcher, error) { chromeLauncher := launcher.New(). - Leakless(false). + Leakless(true). Set("disable-gpu", "true"). Set("ignore-certificate-errors", "true"). Set("ignore-certificate-errors", "1"). From 445b1f5a65c830b87769230bbc6880f9d756187e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:00:27 +0000 Subject: [PATCH 004/183] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.85 to 0.0.86. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.85...v0.0.86) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 907c2756..660cbd48 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.33 github.com/projectdiscovery/retryablehttp-go v1.0.53 github.com/projectdiscovery/useragent v0.0.41 - github.com/projectdiscovery/utils v0.0.85 + github.com/projectdiscovery/utils v0.0.86 github.com/projectdiscovery/wappalyzergo v0.0.115 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index fb7c3d2d..8a135c5d 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.41 h1:GWHPIArnz6/rKpfbqlP484QmHiOFERH0tewvmAh1MHE= github.com/projectdiscovery/useragent v0.0.41/go.mod h1:oXjattkrFK9Y/8c+9/6aBkAA307L/NWQrs28uJaE9ow= -github.com/projectdiscovery/utils v0.0.85 h1:JpCVc9GJwJLNHy1MBPmAHJcE6rs7bRv72Trb3u84OHE= -github.com/projectdiscovery/utils v0.0.85/go.mod h1:ttiPgS2LmLFd+VRBUdgfLKMMdrF98zX7z5W+K71MX40= +github.com/projectdiscovery/utils v0.0.86 h1:sHmomrbKz+fDWbD0Rdrq8nP4Sj/b7wyjECcnsgo1mIA= +github.com/projectdiscovery/utils v0.0.86/go.mod h1:tVAmuSVbMni9GN2wp4wYXR3gSgooq/h3llvEvt4AAaA= github.com/projectdiscovery/wappalyzergo v0.0.115 h1:Xe916MBsliAWwogLHZN0qTrjyCy56PHuGUkdl0DTchk= github.com/projectdiscovery/wappalyzergo v0.0.115/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= From 234789ac57f90c41f26368501d5c37b948837e9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:00:37 +0000 Subject: [PATCH 005/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.48 to 0.0.50 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.48 to 0.0.50. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.48...v0.0.50) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 907c2756..514e4504 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.48 + github.com/projectdiscovery/dsl v0.0.50 github.com/projectdiscovery/fastdialer v0.0.64 github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 diff --git a/go.sum b/go.sum index fb7c3d2d..45ae5ca9 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.48 h1:cdchhF0xofP8pi+HusNcVpw7oUuln7bS7KtzHtaItkg= -github.com/projectdiscovery/dsl v0.0.48/go.mod h1:px2y6PLhdUx77KwPMpk6MBCaQTtTE1PpG4ZVmsgBmK8= +github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM= +github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g= github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw= github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo= github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= From 716abdb8747858b6f3a5df2f1e7acd9c2874cc5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:24:12 +0000 Subject: [PATCH 006/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.41 to 0.0.44. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.41...v0.0.44) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 660cbd48..ff8529ec 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/ratelimit v0.0.33 github.com/projectdiscovery/retryablehttp-go v1.0.53 - github.com/projectdiscovery/useragent v0.0.41 + github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.86 github.com/projectdiscovery/wappalyzergo v0.0.115 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 8a135c5d..25aa1528 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.53 h1:Kp4590eOwFhUH9oIaCFMtudL github.com/projectdiscovery/retryablehttp-go v1.0.53/go.mod h1:b5zNLih0PcfvrDzTRY+QCB1dhFpODQQ2NjI4TtrmRyY= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/useragent v0.0.41 h1:GWHPIArnz6/rKpfbqlP484QmHiOFERH0tewvmAh1MHE= -github.com/projectdiscovery/useragent v0.0.41/go.mod h1:oXjattkrFK9Y/8c+9/6aBkAA307L/NWQrs28uJaE9ow= +github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77hiZE4sIMyt1K4= +github.com/projectdiscovery/useragent v0.0.44/go.mod h1:BrrUQflZxwKxLthzitBX0G2zKPsLL56B354x4b88L3M= github.com/projectdiscovery/utils v0.0.86 h1:sHmomrbKz+fDWbD0Rdrq8nP4Sj/b7wyjECcnsgo1mIA= github.com/projectdiscovery/utils v0.0.86/go.mod h1:tVAmuSVbMni9GN2wp4wYXR3gSgooq/h3llvEvt4AAaA= github.com/projectdiscovery/wappalyzergo v0.0.115 h1:Xe916MBsliAWwogLHZN0qTrjyCy56PHuGUkdl0DTchk= From eb69dc41bdb2b1a24566b48766af81f30e6aac43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:48:16 +0000 Subject: [PATCH 007/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.115 to 0.0.116. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.115...v0.0.116) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cdcbe9e1..e5343eb5 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.53 github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.86 - github.com/projectdiscovery/wappalyzergo v0.0.115 + github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/shirou/gopsutil/v3 v3.23.7 diff --git a/go.sum b/go.sum index 3b9d407b..1855d121 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77h github.com/projectdiscovery/useragent v0.0.44/go.mod h1:BrrUQflZxwKxLthzitBX0G2zKPsLL56B354x4b88L3M= github.com/projectdiscovery/utils v0.0.86 h1:sHmomrbKz+fDWbD0Rdrq8nP4Sj/b7wyjECcnsgo1mIA= github.com/projectdiscovery/utils v0.0.86/go.mod h1:tVAmuSVbMni9GN2wp4wYXR3gSgooq/h3llvEvt4AAaA= -github.com/projectdiscovery/wappalyzergo v0.0.115 h1:Xe916MBsliAWwogLHZN0qTrjyCy56PHuGUkdl0DTchk= -github.com/projectdiscovery/wappalyzergo v0.0.115/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= +github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= +github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From bd858afcf7964fbaf3bf7c3c81faaeaf917d91ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:12:54 +0000 Subject: [PATCH 008/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.53 to 1.0.54. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.53...v1.0.54) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e5343eb5..6dedc976 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/ratelimit v0.0.33 - github.com/projectdiscovery/retryablehttp-go v1.0.53 + github.com/projectdiscovery/retryablehttp-go v1.0.54 github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.86 github.com/projectdiscovery/wappalyzergo v0.0.116 diff --git a/go.sum b/go.sum index 1855d121..39490460 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/projectdiscovery/ratelimit v0.0.33 h1:MT8Oa0VVBBI5w6ZMUJCIIQkjdTVNbzh github.com/projectdiscovery/ratelimit v0.0.33/go.mod h1:Mdbm5Olxd0zddUO3Khy330H1Ei7377/DFIuY9nRZuGM= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.53 h1:Kp4590eOwFhUH9oIaCFMtudLSqL1BX+VNurS42+SC8Q= -github.com/projectdiscovery/retryablehttp-go v1.0.53/go.mod h1:b5zNLih0PcfvrDzTRY+QCB1dhFpODQQ2NjI4TtrmRyY= +github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ= +github.com/projectdiscovery/retryablehttp-go v1.0.54/go.mod h1:J+pg00bYLEgWOZJISi16icHUDbsnkjnA1PmSa2kSMYs= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77hiZE4sIMyt1K4= From 1d51f442583778388d121b043c757834de23b0d8 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Thu, 4 Apr 2024 19:21:56 +0530 Subject: [PATCH 009/183] Fix ignoring host header (#813) --- pkg/engine/standard/crawl.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/engine/standard/crawl.go b/pkg/engine/standard/crawl.go index 1281d45b..933e038b 100644 --- a/pkg/engine/standard/crawl.go +++ b/pkg/engine/standard/crawl.go @@ -41,9 +41,15 @@ func (c *Crawler) makeRequest(s *common.CrawlSession, request *navigation.Reques // Set the headers for the request. for k, v := range request.Headers { req.Header.Set(k, v) + if k == "Host" { + req.Host = v + } } for k, v := range c.Headers { req.Header.Set(k, v) + if k == "Host" { + req.Host = v + } } resp, err := s.HttpClient.Do(req) From 9fc695c83bd24d5aa7f0cf5a5d05deea4c90a5d8 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 5 Apr 2024 16:35:58 +0200 Subject: [PATCH 010/183] adding windows generic --- .goreleaser/windows.yml | 4 +++- pkg/engine/parser/parser_generic.go | 2 +- pkg/engine/parser/{parser_386.go => parser_nojs.go} | 2 +- pkg/utils/jsluice.go | 2 +- pkg/utils/jsluice_test.go | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) rename pkg/engine/parser/{parser_386.go => parser_nojs.go} (96%) diff --git a/.goreleaser/windows.yml b/.goreleaser/windows.yml index f2b1448d..0004e0b2 100644 --- a/.goreleaser/windows.yml +++ b/.goreleaser/windows.yml @@ -15,7 +15,9 @@ builds: goos: - windows goarch: - - amd64 + - 386 + - arm64 + - 386 archives: - format: zip diff --git a/pkg/engine/parser/parser_generic.go b/pkg/engine/parser/parser_generic.go index ce95a79d..831aefc8 100644 --- a/pkg/engine/parser/parser_generic.go +++ b/pkg/engine/parser/parser_generic.go @@ -1,4 +1,4 @@ -//go:build !386 +//go:build !(386 || windows) package parser diff --git a/pkg/engine/parser/parser_386.go b/pkg/engine/parser/parser_nojs.go similarity index 96% rename from pkg/engine/parser/parser_386.go rename to pkg/engine/parser/parser_nojs.go index baa63c83..98e198b6 100644 --- a/pkg/engine/parser/parser_386.go +++ b/pkg/engine/parser/parser_nojs.go @@ -1,4 +1,4 @@ -//go:build 386 +//go:build windows || 386 package parser diff --git a/pkg/utils/jsluice.go b/pkg/utils/jsluice.go index a0f25809..38e5c7b8 100644 --- a/pkg/utils/jsluice.go +++ b/pkg/utils/jsluice.go @@ -1,4 +1,4 @@ -//go:build !386 +//go:build !(386 || windows) package utils diff --git a/pkg/utils/jsluice_test.go b/pkg/utils/jsluice_test.go index c1c87e33..25242a85 100644 --- a/pkg/utils/jsluice_test.go +++ b/pkg/utils/jsluice_test.go @@ -1,4 +1,4 @@ -//go:build !386 +//go:build !(386 || windows) package utils From e511aac2af067c45d37de2b364eb3d39e16dbd37 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 5 Apr 2024 16:41:40 +0200 Subject: [PATCH 011/183] typo --- .goreleaser/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser/windows.yml b/.goreleaser/windows.yml index 0004e0b2..243a5749 100644 --- a/.goreleaser/windows.yml +++ b/.goreleaser/windows.yml @@ -17,7 +17,7 @@ builds: goarch: - 386 - arm64 - - 386 + - amd64 archives: - format: zip From 01c0e3825508b8d527f9448a573edb00138a77cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:52:57 +0000 Subject: [PATCH 012/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.116 to 0.0.117. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.116...v0.0.117) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6dedc976..3aaf882c 100644 --- a/go.mod +++ b/go.mod @@ -21,10 +21,9 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.54 github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.86 - github.com/projectdiscovery/wappalyzergo v0.0.116 + github.com/projectdiscovery/wappalyzergo v0.0.117 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 - github.com/shirou/gopsutil/v3 v3.23.7 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.21.0 @@ -77,6 +76,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sashabaranov/go-openai v1.14.2 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 39490460..74151d67 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77h github.com/projectdiscovery/useragent v0.0.44/go.mod h1:BrrUQflZxwKxLthzitBX0G2zKPsLL56B354x4b88L3M= github.com/projectdiscovery/utils v0.0.86 h1:sHmomrbKz+fDWbD0Rdrq8nP4Sj/b7wyjECcnsgo1mIA= github.com/projectdiscovery/utils v0.0.86/go.mod h1:tVAmuSVbMni9GN2wp4wYXR3gSgooq/h3llvEvt4AAaA= -github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= -github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= +github.com/projectdiscovery/wappalyzergo v0.0.117 h1:x2E7ZDRcGH2zn7dnxXuZ1BxV8bFKvI3bMP8Q9lcUyy8= +github.com/projectdiscovery/wappalyzergo v0.0.117/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From 8ffcdaf82702ec7a32913fcb7d6caaf67b5f3e62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:53:04 +0000 Subject: [PATCH 013/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.50 to 0.0.51 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.50 to 0.0.51. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.50...v0.0.51) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6dedc976..afedfff7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.50 + github.com/projectdiscovery/dsl v0.0.51 github.com/projectdiscovery/fastdialer v0.0.64 github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 @@ -24,7 +24,6 @@ require ( github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 - github.com/shirou/gopsutil/v3 v3.23.7 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 golang.org/x/net v0.21.0 @@ -77,6 +76,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sashabaranov/go-openai v1.14.2 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/smacker/go-tree-sitter v0.0.0-20230720070738-0d0a9f78d8f8 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 39490460..5d6b6627 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM= -github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g= +github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc= +github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg= github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw= github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo= github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= From d721c1a6edcc75b9c5ab8532c87bbc353fdb9e56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:15:35 +0000 Subject: [PATCH 014/183] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.86 to 0.0.88. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.86...v0.0.88) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 +++++--- go.sum | 16 ++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index dae3bb72..f88cb46a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.51 - github.com/projectdiscovery/fastdialer v0.0.64 + github.com/projectdiscovery/fastdialer v0.0.65 github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.33 github.com/projectdiscovery/retryablehttp-go v1.0.54 github.com/projectdiscovery/useragent v0.0.44 - github.com/projectdiscovery/utils v0.0.86 + github.com/projectdiscovery/utils v0.0.88 github.com/projectdiscovery/wappalyzergo v0.0.117 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -50,6 +50,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/kataras/jwt v0.1.8 // indirect @@ -71,7 +72,7 @@ require ( github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/quic-go/quic-go v0.37.7 // indirect + github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -92,6 +93,7 @@ require ( github.com/ysmood/got v0.34.1 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/term v0.17.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index dc060c22..b1fabcbe 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -206,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc= github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg= -github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw= -github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo= +github.com/projectdiscovery/fastdialer v0.0.65 h1:msvKVJyILtP04CXSgSEWv4rUVsk0CCd3xhauo+H82IU= +github.com/projectdiscovery/fastdialer v0.0.65/go.mod h1:wIE10NL7oa/zBCJfr1xAduv3q73aeuGbhfZ1Z8o4NUo= github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= github.com/projectdiscovery/goflags v0.1.45/go.mod h1:JBwA9GYq/7m0a9PTsfRIoPqoB4/+r59MFTVJMDhN0Ic= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -232,12 +234,12 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77hiZE4sIMyt1K4= github.com/projectdiscovery/useragent v0.0.44/go.mod h1:BrrUQflZxwKxLthzitBX0G2zKPsLL56B354x4b88L3M= -github.com/projectdiscovery/utils v0.0.86 h1:sHmomrbKz+fDWbD0Rdrq8nP4Sj/b7wyjECcnsgo1mIA= -github.com/projectdiscovery/utils v0.0.86/go.mod h1:tVAmuSVbMni9GN2wp4wYXR3gSgooq/h3llvEvt4AAaA= +github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs= +github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA= github.com/projectdiscovery/wappalyzergo v0.0.117 h1:x2E7ZDRcGH2zn7dnxXuZ1BxV8bFKvI3bMP8Q9lcUyy8= github.com/projectdiscovery/wappalyzergo v0.0.117/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= -github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= -github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= @@ -348,6 +350,8 @@ github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGj github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= +github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= From 6035dc3277edaeb60958ee47f98fd1ac0842617a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 13:37:51 +0000 Subject: [PATCH 015/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.33 to 0.0.36. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.33...v0.0.36) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f88cb46a..f2d1f553 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.16 - github.com/projectdiscovery/ratelimit v0.0.33 + github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.54 github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.88 diff --git a/go.sum b/go.sum index b1fabcbe..70396f01 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclg github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.33 h1:MT8Oa0VVBBI5w6ZMUJCIIQkjdTVNbzhGRDMrNqV1BQ4= -github.com/projectdiscovery/ratelimit v0.0.33/go.mod h1:Mdbm5Olxd0zddUO3Khy330H1Ei7377/DFIuY9nRZuGM= +github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E4C66J/wIm/ow= +github.com/projectdiscovery/ratelimit v0.0.36/go.mod h1:Y9xgLZ3ttnp144OLeL1bzGukieNF5wY0dVdigi+PrZ4= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ= From 78a559611f1cb0ac79f082444a9325f3aa3d86d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:00:55 +0000 Subject: [PATCH 016/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.54 to 1.0.55. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.54...v1.0.55) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2d1f553..e286bd0b 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/ratelimit v0.0.36 - github.com/projectdiscovery/retryablehttp-go v1.0.54 + github.com/projectdiscovery/retryablehttp-go v1.0.55 github.com/projectdiscovery/useragent v0.0.44 github.com/projectdiscovery/utils v0.0.88 github.com/projectdiscovery/wappalyzergo v0.0.117 diff --git a/go.sum b/go.sum index 70396f01..249eb98f 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E github.com/projectdiscovery/ratelimit v0.0.36/go.mod h1:Y9xgLZ3ttnp144OLeL1bzGukieNF5wY0dVdigi+PrZ4= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ= -github.com/projectdiscovery/retryablehttp-go v1.0.54/go.mod h1:J+pg00bYLEgWOZJISi16icHUDbsnkjnA1PmSa2kSMYs= +github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls= +github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77hiZE4sIMyt1K4= From bc4f29b9416fe38a42c7f46e832abca6caa76fa6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:51:30 +0000 Subject: [PATCH 017/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.65 to 0.0.67. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.65...v0.0.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e286bd0b..c7a4d86c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.51 - github.com/projectdiscovery/fastdialer v0.0.65 + github.com/projectdiscovery/fastdialer v0.0.67 github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 diff --git a/go.sum b/go.sum index 249eb98f..905f8d3a 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc= github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg= -github.com/projectdiscovery/fastdialer v0.0.65 h1:msvKVJyILtP04CXSgSEWv4rUVsk0CCd3xhauo+H82IU= -github.com/projectdiscovery/fastdialer v0.0.65/go.mod h1:wIE10NL7oa/zBCJfr1xAduv3q73aeuGbhfZ1Z8o4NUo= +github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= +github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= github.com/projectdiscovery/goflags v0.1.45/go.mod h1:JBwA9GYq/7m0a9PTsfRIoPqoB4/+r59MFTVJMDhN0Ic= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= From f3ca2be298d146f492295cd5ade7eed4100f74e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:51:46 +0000 Subject: [PATCH 018/183] chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.1.16 to 1.1.34. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.1.16...v1.1.34) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e286bd0b..8ab35aff 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 - github.com/projectdiscovery/mapcidr v1.1.16 + github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.55 github.com/projectdiscovery/useragent v0.0.44 diff --git a/go.sum b/go.sum index 249eb98f..ce386c3c 100644 --- a/go.sum +++ b/go.sum @@ -220,8 +220,8 @@ github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3 github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo= -github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= +github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= +github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E4C66J/wIm/ow= From e2848656ec2b4338e31d8e283880d5d75a7e71f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:13:49 +0000 Subject: [PATCH 019/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.51 to 0.0.52 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.51 to 0.0.52. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.51...v0.0.52) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c7a4d86c..c01f0aae 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.51 + github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.67 github.com/projectdiscovery/goflags v0.1.45 github.com/projectdiscovery/gologger v1.1.12 diff --git a/go.sum b/go.sum index 905f8d3a..b3f0df62 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc= -github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg= +github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= +github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= From b517db607578783527e3eae3d16d4d55f2fe6fa1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:14:58 +0000 Subject: [PATCH 020/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.44 to 0.0.46. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.44...v0.0.46) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 892f5870..ddf11c23 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.51 github.com/projectdiscovery/fastdialer v0.0.67 - github.com/projectdiscovery/goflags v0.1.45 + github.com/projectdiscovery/goflags v0.1.47 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.55 - github.com/projectdiscovery/useragent v0.0.44 + github.com/projectdiscovery/useragent v0.0.46 github.com/projectdiscovery/utils v0.0.88 github.com/projectdiscovery/wappalyzergo v0.0.117 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 9f77cc3b..a756dea6 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4 github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg= github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= -github.com/projectdiscovery/goflags v0.1.45 h1:quDCIWRsULQHmDI1UvXptEn9cSRCeiHWT3mxB6bpxnA= -github.com/projectdiscovery/goflags v0.1.45/go.mod h1:JBwA9GYq/7m0a9PTsfRIoPqoB4/+r59MFTVJMDhN0Ic= +github.com/projectdiscovery/goflags v0.1.47 h1:cO0m+Xl4kXvwlyN4Yp61te+Utf3y2IiLXB1JRLvdFMY= +github.com/projectdiscovery/goflags v0.1.47/go.mod h1:+JKmFaqKtFtDYyhmfKEAk5BqnGRQuSF0fR+nk2r9oBQ= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/useragent v0.0.44 h1:5d4aLEuhqJTx4TzUVTdjwu55pYwC77hiZE4sIMyt1K4= -github.com/projectdiscovery/useragent v0.0.44/go.mod h1:BrrUQflZxwKxLthzitBX0G2zKPsLL56B354x4b88L3M= +github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0+h+EosNc0Dpk= +github.com/projectdiscovery/useragent v0.0.46/go.mod h1:fQ+O3EasUxyQDPpXt2BB4/x1GSWsnManWKP/XQX8TsM= github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs= github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA= github.com/projectdiscovery/wappalyzergo v0.0.117 h1:x2E7ZDRcGH2zn7dnxXuZ1BxV8bFKvI3bMP8Q9lcUyy8= From f037c2564e459f162f050e4d8065fccf9fdb5bbf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:38:51 +0000 Subject: [PATCH 021/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.55 to 1.0.56. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.55...v1.0.56) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b1df351..a8f7d907 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 - github.com/projectdiscovery/retryablehttp-go v1.0.55 + github.com/projectdiscovery/retryablehttp-go v1.0.56 github.com/projectdiscovery/useragent v0.0.46 github.com/projectdiscovery/utils v0.0.88 github.com/projectdiscovery/wappalyzergo v0.0.117 diff --git a/go.sum b/go.sum index 3f376e6b..c9870385 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E github.com/projectdiscovery/ratelimit v0.0.36/go.mod h1:Y9xgLZ3ttnp144OLeL1bzGukieNF5wY0dVdigi+PrZ4= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls= -github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng= +github.com/projectdiscovery/retryablehttp-go v1.0.56 h1:zz5QhDGmhw/NwcDSDbjkkna7tkb311J5kLDl79rDnVc= +github.com/projectdiscovery/retryablehttp-go v1.0.56/go.mod h1:eHn/R1qUQUN0yR7UjA8H13EXILp5OPq1dHFHBXj9w10= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0+h+EosNc0Dpk= From 6d6db4308479816d21229155d3ec49f9ebc0e9df Mon Sep 17 00:00:00 2001 From: eveneast <166489430+eveneast@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:23:13 +0800 Subject: [PATCH 022/183] Fix some comments (#851) * Update README.md * Fix some comments Signed-off-by: eveneast --------- Signed-off-by: eveneast Co-authored-by: zy9ard3 <67743789+zy9ard3@users.noreply.github.com> Co-authored-by: Sandeep Singh --- README.md | 1 + pkg/engine/common/http.go | 2 +- pkg/engine/parser/parser.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9bef2334..2011262e 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Flags: INPUT: -u, -list string[] target url / list to crawl -resume string resume scan using resume.cfg + -e, -exclude string[] exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex) CONFIGURATION: -r, -resolvers string[] list of custom resolver (file or comma separated) diff --git a/pkg/engine/common/http.go b/pkg/engine/common/http.go index 87b25ddc..0e143713 100644 --- a/pkg/engine/common/http.go +++ b/pkg/engine/common/http.go @@ -19,7 +19,7 @@ import ( type RedirectCallback func(resp *http.Response, depth int) -// BuildClient builds a http client based on a profile +// BuildHttpClient builds a http client based on a profile func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirectCallback RedirectCallback) (*retryablehttp.Client, *fastdialer.Dialer, error) { // Single Host retryablehttpOptions := retryablehttp.DefaultOptionsSingle diff --git a/pkg/engine/parser/parser.go b/pkg/engine/parser/parser.go index d621a6de..9b068e75 100644 --- a/pkg/engine/parser/parser.go +++ b/pkg/engine/parser/parser.go @@ -397,7 +397,7 @@ func bodyVideoTagParser(resp *navigation.Response) (navigationRequests []*naviga return } -// bodyButtonFormactionTagParser parses blockquote cite tag from response +// bodyBlockquoteCiteTagParser parses blockquote cite tag from response func bodyBlockquoteCiteTagParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { resp.Reader.Find("blockquote[cite]").Each(func(i int, item *goquery.Selection) { src, ok := item.Attr("cite") From 21a19a7540ce9faba9a8407eb65ef5581bb5fe52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 16 Apr 2024 14:59:57 +0300 Subject: [PATCH 023/183] skip scope validation for intial host --- pkg/engine/common/base.go | 13 ++++++++++--- pkg/navigation/request.go | 23 ++++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pkg/engine/common/base.go b/pkg/engine/common/base.go index 76bbf8cc..3fff4da7 100644 --- a/pkg/engine/common/base.go +++ b/pkg/engine/common/base.go @@ -145,7 +145,7 @@ func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error) { cancel() return nil, err } - queue.Push(&navigation.Request{Method: http.MethodGet, URL: URL, Depth: 0}, 0) + queue.Push(&navigation.Request{Method: http.MethodGet, URL: URL, Depth: 0, SkipValidation: true}, 0) if s.KnownFiles != nil { navigationRequests, err := s.KnownFiles.Request(URL) @@ -205,7 +205,12 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { continue } - if ok, err := s.Options.ValidateScope(req.URL, crawlSession.Hostname); err != nil || !ok { + inScope, scopeErr := s.Options.ValidateScope(req.URL, crawlSession.Hostname) + if scopeErr != nil { + gologger.Debug().Msgf("Error validating scope for `%v`: %v. skipping", req.URL, scopeErr) + continue + } + if !req.SkipValidation && !inScope { gologger.Debug().Msgf("`%v` not in scope. skipping", req.URL) continue } @@ -224,7 +229,9 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { resp, err := doRequest(crawlSession, req) - s.Output(req, resp, nil, err) + if inScope { + s.Output(req, resp, nil, err) + } if err != nil { gologger.Warning().Msgf("Could not request seed URL %s: %s\n", req.URL, err) diff --git a/pkg/navigation/request.go b/pkg/navigation/request.go index f427d0d0..dcd9c96e 100644 --- a/pkg/navigation/request.go +++ b/pkg/navigation/request.go @@ -10,17 +10,18 @@ type Depth struct{} // Request is a navigation request for the crawler type Request struct { - Method string `json:"method,omitempty"` - URL string `json:"endpoint,omitempty"` - Body string `json:"body,omitempty"` - Depth int `json:"-"` - Headers map[string]string `json:"headers,omitempty"` - Tag string `json:"tag,omitempty"` - Attribute string `json:"attribute,omitempty"` - RootHostname string `json:"-"` - Source string `json:"source,omitempty"` - CustomFields map[string][]string `json:"-"` - Raw string `json:"raw,omitempty"` + Method string `json:"method,omitempty"` + URL string `json:"endpoint,omitempty"` + Body string `json:"body,omitempty"` + Depth int `json:"-"` + SkipValidation bool `json:"-"` + Headers map[string]string `json:"headers,omitempty"` + Tag string `json:"tag,omitempty"` + Attribute string `json:"attribute,omitempty"` + RootHostname string `json:"-"` + Source string `json:"source,omitempty"` + CustomFields map[string][]string `json:"-"` + Raw string `json:"raw,omitempty"` } // RequestURL returns the request URL for the navigation From 26f87d8d619562ebd372433e9cfb9d52f984ce9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 18 Apr 2024 12:11:06 +0300 Subject: [PATCH 024/183] add no-clobber flag --- cmd/katana/main.go | 1 + go.mod | 1 + go.sum | 8 +++++ pkg/output/options.go | 1 + pkg/output/output.go | 60 ++++++++++++++++++++++++++++++++++-- pkg/types/crawler_options.go | 1 + pkg/types/options.go | 2 ++ 7 files changed, 72 insertions(+), 2 deletions(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 0d4ac4e7..8d213f24 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -168,6 +168,7 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), + flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"), flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"), flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"), flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"), diff --git a/go.mod b/go.mod index a8f7d907..2ca2f77f 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.67 + github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/goflags v0.1.47 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 diff --git a/go.sum b/go.sum index c9870385..2334874f 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,7 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= @@ -210,6 +211,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= +github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= +github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goflags v0.1.47 h1:cO0m+Xl4kXvwlyN4Yp61te+Utf3y2IiLXB1JRLvdFMY= github.com/projectdiscovery/goflags v0.1.47/go.mod h1:+JKmFaqKtFtDYyhmfKEAk5BqnGRQuSF0fR+nk2r9oBQ= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -282,6 +285,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -402,6 +406,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -440,6 +445,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -452,6 +458,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= @@ -463,6 +470,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/pkg/output/options.go b/pkg/output/options.go index 03e1bb37..d519c353 100644 --- a/pkg/output/options.go +++ b/pkg/output/options.go @@ -12,6 +12,7 @@ type Options struct { JSON bool Verbose bool StoreResponse bool + NoClobber bool OmitRaw bool OmitBody bool OutputFile string diff --git a/pkg/output/output.go b/pkg/output/output.go index e879ea43..53025e89 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "sync" @@ -13,6 +14,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/mitchellh/mapstructure" "github.com/projectdiscovery/dsl" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/katana/pkg/utils/extensions" errorutil "github.com/projectdiscovery/utils/errors" @@ -48,6 +50,7 @@ type StandardWriter struct { outputMutex *sync.Mutex storeResponse bool storeResponseDir string + noClobber bool omitRaw bool omitBody bool errorFile *fileWriter @@ -68,6 +71,7 @@ func New(options Options) (Writer, error) { outputMutex: &sync.Mutex{}, storeResponse: options.StoreResponse, storeResponseDir: options.StoreResponseDir, + noClobber: options.NoClobber, omitRaw: options.OmitRaw, omitBody: options.OmitBody, matchRegex: options.MatchRegex, @@ -117,8 +121,13 @@ func New(options Options) (Writer, error) { if options.StoreResponseDir != DefaultResponseDir && options.StoreResponseDir != "" { writer.storeResponseDir = options.StoreResponseDir } - _ = os.RemoveAll(writer.storeResponseDir) - _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + if options.NoClobber { + writer.storeResponseDir = createDirNameNoClobber(writer.storeResponseDir) + _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + } else { + removeDirsWithSuffix(writer.storeResponseDir) + _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + } // todo: the index file seems never used? _, err := newFileOutputWriter(filepath.Join(writer.storeResponseDir, indexFile)) if err != nil { @@ -252,6 +261,53 @@ func (w *StandardWriter) Close() error { return nil } +func createDirNameNoClobber(dir string) string { + if !fileutil.FolderExists(dir) { + return dir + } + + parentDir, dirName := filepath.Dir(dir), filepath.Base(dir) + entries, err := os.ReadDir(parentDir) + if err != nil { + return dirName + } + + highestNum := 0 + regex := regexp.MustCompile(fmt.Sprintf("^%s(\\d+)$", regexp.QuoteMeta(dirName))) + for _, entry := range entries { + if entry.IsDir() { + name := entry.Name() + matches := regex.FindStringSubmatch(name) + if matches != nil { + if num, err := strconv.Atoi(matches[1]); err == nil && num > highestNum { + highestNum = num + } + } + } + } + + newDirName := fmt.Sprintf("%s%d", dirName, highestNum+1) + newFullPath := fmt.Sprintf("%s/%s", parentDir, newDirName) + return newFullPath +} + +func removeDirsWithSuffix(dir string) { + parentDir, dirName := filepath.Dir(dir), filepath.Base(dir) + entries, _ := os.ReadDir(parentDir) + + pattern := fmt.Sprintf("^%s(\\d*)$", regexp.QuoteMeta(dirName)) + regex := regexp.MustCompile(pattern) + for _, entry := range entries { + if entry.IsDir() { + name := entry.Name() + if regex.MatchString(name) { + fullPath := filepath.Join(parentDir, name) + _ = os.RemoveAll(fullPath) + } + } + } +} + // matchOutput checks if the event matches the output regex func (w *StandardWriter) matchOutput(event *Result) bool { if w.matchRegex == nil && w.outputMatchCondition == "" { diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index b03b53de..cf474d9f 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -72,6 +72,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { Fields: options.Fields, StoreFields: options.StoreFields, StoreResponseDir: options.StoreResponseDir, + NoClobber: options.NoClobber, OmitRaw: options.OmitRaw, OmitBody: options.OmitBody, FieldConfig: options.FieldConfig, diff --git a/pkg/types/options.go b/pkg/types/options.go index af6bc6fb..5743be82 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -114,6 +114,8 @@ type Options struct { StoreResponse bool // StoreResponseDir specifies if katana should use a custom directory to store http requests/responses StoreResponseDir string + // NoClobber specifies if katana should overwrite existing output files + NoClobber bool // OmitRaw omits raw requests/responses from the output OmitRaw bool // OmitBody omits the response body from the output From d5af7140a369207095951aa73f77d45ab6f949dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:18:37 +0000 Subject: [PATCH 025/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.47 to 0.1.49. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.47...v0.1.49) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index a8f7d907..8a46b27d 100644 --- a/go.mod +++ b/go.mod @@ -13,20 +13,20 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.67 - github.com/projectdiscovery/goflags v0.1.47 + github.com/projectdiscovery/goflags v0.1.49 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.56 github.com/projectdiscovery/useragent v0.0.46 - github.com/projectdiscovery/utils v0.0.88 + github.com/projectdiscovery/utils v0.0.89 github.com/projectdiscovery/wappalyzergo v0.0.117 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.21.0 + golang.org/x/net v0.23.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -95,7 +95,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/term v0.18.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) @@ -138,10 +138,10 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index c9870385..ded982bd 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= -github.com/projectdiscovery/goflags v0.1.47 h1:cO0m+Xl4kXvwlyN4Yp61te+Utf3y2IiLXB1JRLvdFMY= -github.com/projectdiscovery/goflags v0.1.47/go.mod h1:+JKmFaqKtFtDYyhmfKEAk5BqnGRQuSF0fR+nk2r9oBQ= +github.com/projectdiscovery/goflags v0.1.49 h1:0e9wya431WDeVm8ZtlyqBQ+rwnhDjUswDMcS0did9Tg= +github.com/projectdiscovery/goflags v0.1.49/go.mod h1:f0zRbaa5QLrjfJQ5v0efvq8EhkDGhCm9h0hsahjjKFc= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -234,8 +234,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0+h+EosNc0Dpk= github.com/projectdiscovery/useragent v0.0.46/go.mod h1:fQ+O3EasUxyQDPpXt2BB4/x1GSWsnManWKP/XQX8TsM= -github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs= -github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA= +github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= +github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= github.com/projectdiscovery/wappalyzergo v0.0.117 h1:x2E7ZDRcGH2zn7dnxXuZ1BxV8bFKvI3bMP8Q9lcUyy8= github.com/projectdiscovery/wappalyzergo v0.0.117/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= @@ -379,8 +379,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -406,8 +406,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -447,16 +447,16 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 33fc9152bc3cb0077220afe09c7d5765a64f4d24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:40:37 +0000 Subject: [PATCH 026/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.67 to 0.0.68. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.67...v0.0.68) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a46b27d..307bf788 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 - github.com/projectdiscovery/fastdialer v0.0.67 + github.com/projectdiscovery/fastdialer v0.0.68 github.com/projectdiscovery/goflags v0.1.49 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 diff --git a/go.sum b/go.sum index ded982bd..01763934 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= -github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= +github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo= +github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M= github.com/projectdiscovery/goflags v0.1.49 h1:0e9wya431WDeVm8ZtlyqBQ+rwnhDjUswDMcS0did9Tg= github.com/projectdiscovery/goflags v0.1.49/go.mod h1:f0zRbaa5QLrjfJQ5v0efvq8EhkDGhCm9h0hsahjjKFc= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= From 552bff27cc943ea1a3393588e76c9d0a32d2c9aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:40:38 +0000 Subject: [PATCH 027/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.56 to 1.0.57. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.56...v1.0.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a46b27d..aa10b22a 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 - github.com/projectdiscovery/retryablehttp-go v1.0.56 + github.com/projectdiscovery/retryablehttp-go v1.0.57 github.com/projectdiscovery/useragent v0.0.46 github.com/projectdiscovery/utils v0.0.89 github.com/projectdiscovery/wappalyzergo v0.0.117 diff --git a/go.sum b/go.sum index ded982bd..bd7df27b 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E github.com/projectdiscovery/ratelimit v0.0.36/go.mod h1:Y9xgLZ3ttnp144OLeL1bzGukieNF5wY0dVdigi+PrZ4= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.56 h1:zz5QhDGmhw/NwcDSDbjkkna7tkb311J5kLDl79rDnVc= -github.com/projectdiscovery/retryablehttp-go v1.0.56/go.mod h1:eHn/R1qUQUN0yR7UjA8H13EXILp5OPq1dHFHBXj9w10= +github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY= +github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0+h+EosNc0Dpk= From aafcfbe5e8b8c425bffcf8aba71176680a0e7ca0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 12:41:03 +0000 Subject: [PATCH 028/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.117 to 0.0.119. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.117...v0.0.119) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 8a46b27d..47de8a1e 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.56 github.com/projectdiscovery/useragent v0.0.46 github.com/projectdiscovery/utils v0.0.89 - github.com/projectdiscovery/wappalyzergo v0.0.117 + github.com/projectdiscovery/wappalyzergo v0.0.119 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.23.0 + golang.org/x/net v0.24.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -95,7 +95,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/term v0.19.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) @@ -138,10 +138,10 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index ded982bd..f554a654 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0 github.com/projectdiscovery/useragent v0.0.46/go.mod h1:fQ+O3EasUxyQDPpXt2BB4/x1GSWsnManWKP/XQX8TsM= github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= -github.com/projectdiscovery/wappalyzergo v0.0.117 h1:x2E7ZDRcGH2zn7dnxXuZ1BxV8bFKvI3bMP8Q9lcUyy8= -github.com/projectdiscovery/wappalyzergo v0.0.117/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= +github.com/projectdiscovery/wappalyzergo v0.0.119 h1:FklfPD/i/H93IuUpTTw8DLLI9pchLHVzeluNtUS3VGc= +github.com/projectdiscovery/wappalyzergo v0.0.119/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -379,8 +379,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -406,8 +406,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -447,16 +447,16 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 87ba45bcfa21859f7bc44da8f65e85957d9c2f1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:02:53 +0000 Subject: [PATCH 029/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.46 to 0.0.47. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.46...v0.0.47) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 98ffb18e..7f6e50b6 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.57 - github.com/projectdiscovery/useragent v0.0.46 + github.com/projectdiscovery/useragent v0.0.47 github.com/projectdiscovery/utils v0.0.89 github.com/projectdiscovery/wappalyzergo v0.0.119 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 3ec9dfac..06da8c87 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaD github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/useragent v0.0.46 h1:fmIA8oK04ukVCkHv1bGCeAxzWyxYRX0+h+EosNc0Dpk= -github.com/projectdiscovery/useragent v0.0.46/go.mod h1:fQ+O3EasUxyQDPpXt2BB4/x1GSWsnManWKP/XQX8TsM= +github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= +github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= github.com/projectdiscovery/wappalyzergo v0.0.119 h1:FklfPD/i/H93IuUpTTw8DLLI9pchLHVzeluNtUS3VGc= From da7c99183b7e9e507e6bf88f49ff49a59b2c1bc1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 29 Apr 2024 14:29:13 +0200 Subject: [PATCH 030/183] deprecated pkg --- go.mod | 9 ++++----- go.sum | 24 ++++++++---------------- pkg/output/output.go | 2 +- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 2ca2f77f..560667df 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.67 - github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/goflags v0.1.47 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 @@ -27,7 +26,7 @@ require ( github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.21.0 + golang.org/x/net v0.24.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -96,7 +95,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/term v0.19.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) @@ -139,10 +138,10 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 2334874f..cf69f876 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= @@ -211,8 +210,6 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E= github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84= -github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= -github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goflags v0.1.47 h1:cO0m+Xl4kXvwlyN4Yp61te+Utf3y2IiLXB1JRLvdFMY= github.com/projectdiscovery/goflags v0.1.47/go.mod h1:+JKmFaqKtFtDYyhmfKEAk5BqnGRQuSF0fR+nk2r9oBQ= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -285,7 +282,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -383,8 +379,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -406,13 +402,12 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -445,7 +440,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -453,24 +447,22 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/pkg/output/output.go b/pkg/output/output.go index 53025e89..839b41c1 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -14,10 +14,10 @@ import ( "github.com/logrusorgru/aurora" "github.com/mitchellh/mapstructure" "github.com/projectdiscovery/dsl" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/katana/pkg/utils/extensions" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" ) const ( From 485e250c0b02039ac0f666f3ec395e118a162845 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 29 Apr 2024 14:47:05 +0200 Subject: [PATCH 031/183] win compatibility --- pkg/output/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/output/output.go b/pkg/output/output.go index 839b41c1..90cb65ee 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -287,7 +287,7 @@ func createDirNameNoClobber(dir string) string { } newDirName := fmt.Sprintf("%s%d", dirName, highestNum+1) - newFullPath := fmt.Sprintf("%s/%s", parentDir, newDirName) + newFullPath := filepath.Join(parentDir, newDirName) return newFullPath } From 39e28aa1ac835f5ac16a766c6a501fb7cb3a44f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 12:54:51 +0000 Subject: [PATCH 032/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.119 to 0.0.120. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.119...v0.0.120) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7f6e50b6..663a14ca 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.57 github.com/projectdiscovery/useragent v0.0.47 github.com/projectdiscovery/utils v0.0.89 - github.com/projectdiscovery/wappalyzergo v0.0.119 + github.com/projectdiscovery/wappalyzergo v0.0.120 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 06da8c87..45ded06b 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= -github.com/projectdiscovery/wappalyzergo v0.0.119 h1:FklfPD/i/H93IuUpTTw8DLLI9pchLHVzeluNtUS3VGc= -github.com/projectdiscovery/wappalyzergo v0.0.119/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= +github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From 67e8676b26c2e4354fcf7e0169a7d5fe42416d16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:21:42 +0000 Subject: [PATCH 033/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.36 to 0.0.39. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.36...v0.0.39) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 663a14ca..ab33c892 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,10 @@ require ( github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.36 + github.com/projectdiscovery/ratelimit v0.0.39 github.com/projectdiscovery/retryablehttp-go v1.0.57 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.89 + github.com/projectdiscovery/utils v0.0.90 github.com/projectdiscovery/wappalyzergo v0.0.120 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 45ded06b..89800db1 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.36 h1:uhzft5Aci9RhDfKQNfqNpt8xAiB+b7E4C66J/wIm/ow= -github.com/projectdiscovery/ratelimit v0.0.36/go.mod h1:Y9xgLZ3ttnp144OLeL1bzGukieNF5wY0dVdigi+PrZ4= +github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc= +github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY= @@ -234,8 +234,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= -github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= +github.com/projectdiscovery/utils v0.0.90 h1:ZBClkrtC6U2xuT8AI6/MrYgm3cxQnwSZmM8/tIlA0Yk= +github.com/projectdiscovery/utils v0.0.90/go.mod h1:laZCMcUTT1vTmclUCW+zpi/fjeYsgU1gMwgv2CxNcRk= github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 63dcc7b3529f2d16e119bed3988d13fa92370b8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:21:42 +0000 Subject: [PATCH 034/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.68 to 0.0.69. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.68...v0.0.69) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 663a14ca..a356e4d7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 - github.com/projectdiscovery/fastdialer v0.0.68 + github.com/projectdiscovery/fastdialer v0.0.69 github.com/projectdiscovery/goflags v0.1.49 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.36 github.com/projectdiscovery/retryablehttp-go v1.0.57 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.89 + github.com/projectdiscovery/utils v0.0.90 github.com/projectdiscovery/wappalyzergo v0.0.120 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 45ded06b..991e1c0e 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo= -github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M= +github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= +github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= github.com/projectdiscovery/goflags v0.1.49 h1:0e9wya431WDeVm8ZtlyqBQ+rwnhDjUswDMcS0did9Tg= github.com/projectdiscovery/goflags v0.1.49/go.mod h1:f0zRbaa5QLrjfJQ5v0efvq8EhkDGhCm9h0hsahjjKFc= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -234,8 +234,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w= -github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI= +github.com/projectdiscovery/utils v0.0.90 h1:ZBClkrtC6U2xuT8AI6/MrYgm3cxQnwSZmM8/tIlA0Yk= +github.com/projectdiscovery/utils v0.0.90/go.mod h1:laZCMcUTT1vTmclUCW+zpi/fjeYsgU1gMwgv2CxNcRk= github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 435284396493a4bbafe373397eb02cc958b2a61a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:47:58 +0000 Subject: [PATCH 035/183] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.89 to 0.0.91. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.89...v0.0.91) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1c4df3ef..62e0692e 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.39 github.com/projectdiscovery/retryablehttp-go v1.0.57 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.90 + github.com/projectdiscovery/utils v0.0.91 github.com/projectdiscovery/wappalyzergo v0.0.120 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index d4e128cc..39a60b9c 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.90 h1:ZBClkrtC6U2xuT8AI6/MrYgm3cxQnwSZmM8/tIlA0Yk= -github.com/projectdiscovery/utils v0.0.90/go.mod h1:laZCMcUTT1vTmclUCW+zpi/fjeYsgU1gMwgv2CxNcRk= +github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w= +github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls= github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 10fee8250429dfff63301d75031446a5abd34283 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:48:13 +0000 Subject: [PATCH 036/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.49 to 0.1.50. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.49...v0.1.50) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1c4df3ef..5aa2ce88 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.69 - github.com/projectdiscovery/goflags v0.1.49 + github.com/projectdiscovery/goflags v0.1.50 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index d4e128cc..105cabc4 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= -github.com/projectdiscovery/goflags v0.1.49 h1:0e9wya431WDeVm8ZtlyqBQ+rwnhDjUswDMcS0did9Tg= -github.com/projectdiscovery/goflags v0.1.49/go.mod h1:f0zRbaa5QLrjfJQ5v0efvq8EhkDGhCm9h0hsahjjKFc= +github.com/projectdiscovery/goflags v0.1.50 h1:vhpM3mDTnXTktypUuVKWxgRpuBFJE+wzOr56rnOl4Bc= +github.com/projectdiscovery/goflags v0.1.50/go.mod h1:3A8u7Q7e59S7f6QlJmMvKMXD/Tp+bfWre64gUpt6a6Q= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From 81254ebb7f0d87a88a0242b784929b30d4cb4019 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 3 May 2024 15:39:33 +0200 Subject: [PATCH 037/183] using ptr --- pkg/types/crawler_options.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index b03b53de..8186780d 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -24,7 +24,7 @@ type CrawlerOptions struct { // OutputWriter is the interface for writing output OutputWriter output.Writer // RateLimit is a mechanism for controlling request rate limit - RateLimit ratelimit.Limiter + RateLimit *ratelimit.Limiter // Options contains the user specified configuration options Options *Options // ExtensionsValidator is a validator for file extensions @@ -113,9 +113,9 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { } if options.RateLimit > 0 { - crawlerOptions.RateLimit = *ratelimit.New(context.Background(), uint(options.RateLimit), time.Second) + crawlerOptions.RateLimit = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second) } else if options.RateLimitMinute > 0 { - crawlerOptions.RateLimit = *ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute) + crawlerOptions.RateLimit = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute) } wappalyze, err := wappalyzer.New() From 17dc30fdfa8a3e46b5ab4f8e9b65088d80da830c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 12:23:50 +0000 Subject: [PATCH 038/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.50 to 0.1.51. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.50...v0.1.51) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 08eac852..72ada866 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.69 - github.com/projectdiscovery/goflags v0.1.50 + github.com/projectdiscovery/goflags v0.1.51 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index 22b30a66..474f454e 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= -github.com/projectdiscovery/goflags v0.1.50 h1:vhpM3mDTnXTktypUuVKWxgRpuBFJE+wzOr56rnOl4Bc= -github.com/projectdiscovery/goflags v0.1.50/go.mod h1:3A8u7Q7e59S7f6QlJmMvKMXD/Tp+bfWre64gUpt6a6Q= +github.com/projectdiscovery/goflags v0.1.51 h1:PhMekTX727L1YqBfP0of0clSygrq20RnWWp0+khAxqo= +github.com/projectdiscovery/goflags v0.1.51/go.mod h1:muJxbcOFi4kzg0G30S526X6dY/OyuDInchYIkoZOudU= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From a36e524a49367c91152fc7001f9490f43fe6132b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 12:23:53 +0000 Subject: [PATCH 039/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.58 to 1.0.59. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.58...v1.0.59) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 08eac852..fb619cb9 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.39 - github.com/projectdiscovery/retryablehttp-go v1.0.57 + github.com/projectdiscovery/retryablehttp-go v1.0.58 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.91 + github.com/projectdiscovery/utils v0.0.92 github.com/projectdiscovery/wappalyzergo v0.0.120 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -122,7 +122,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.58 // indirect + github.com/projectdiscovery/retryabledns v1.0.59 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 22b30a66..5f9bdd9f 100644 --- a/go.sum +++ b/go.sum @@ -226,16 +226,16 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc= github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0= -github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= -github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY= -github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA= +github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= +github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= +github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= +github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w= -github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls= +github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= +github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 445a2a681f7c7b25caa10e49a0fbc7634d6ff140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 12:52:40 +0000 Subject: [PATCH 040/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.57 to 1.0.59. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.57...v1.0.59) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 57b423e5..4b776f02 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.39 - github.com/projectdiscovery/retryablehttp-go v1.0.58 + github.com/projectdiscovery/retryablehttp-go v1.0.59 github.com/projectdiscovery/useragent v0.0.47 github.com/projectdiscovery/utils v0.0.92 github.com/projectdiscovery/wappalyzergo v0.0.120 diff --git a/go.sum b/go.sum index 7d480d2d..b049beaf 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= -github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= +github.com/projectdiscovery/retryablehttp-go v1.0.59 h1:jyeBhFG4iKpJPurlQAlVLgCImTMjwUBaJmZoz90SnVw= +github.com/projectdiscovery/retryablehttp-go v1.0.59/go.mod h1:/L7lD+gOM/44r1mWgwf3knG9Q6gUbNEtbd1DNEq1+bE= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= From ed7f5c3bb30b21a9341fb64ae0596255e2809fe0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 13:20:59 +0000 Subject: [PATCH 041/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.120 to 0.0.121. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.120...v0.0.121) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4b776f02..754d8b46 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.59 github.com/projectdiscovery/useragent v0.0.47 github.com/projectdiscovery/utils v0.0.92 - github.com/projectdiscovery/wappalyzergo v0.0.120 + github.com/projectdiscovery/wappalyzergo v0.0.121 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index b049beaf..d0190144 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= -github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960= -github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= +github.com/projectdiscovery/wappalyzergo v0.0.121/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From f2513e129cdd8a8ee56f37bb17305cd91f26fc7c Mon Sep 17 00:00:00 2001 From: shubo6 <814183583@qq.com> Date: Wed, 8 May 2024 16:09:29 +0800 Subject: [PATCH 042/183] fix the error of using path as filepath change path.Join => filepath.Join --- pkg/output/fields.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/output/fields.go b/pkg/output/fields.go index 72eefe56..ab8666b7 100644 --- a/pkg/output/fields.go +++ b/pkg/output/fields.go @@ -5,6 +5,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "github.com/projectdiscovery/gologger" @@ -84,7 +85,7 @@ func storeFields(output *Result, storeFields []string) { } func appendToFileField(parsed *url.URL, field, data string) { - file, err := os.OpenFile(path.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { return } From 7c1e11332631e286a48ad7720cdab158deb3ad24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 9 May 2024 17:48:11 +0300 Subject: [PATCH 043/183] deduplicate lines in files in store-field-dir --- cmd/katana/main.go | 52 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 0d4ac4e7..ff415265 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "fmt" "math" "os" @@ -68,13 +69,62 @@ func main() { gologger.Fatal().Msgf("could not execute crawling: %s", err) } - // on successful execution remove the resume file in case it exists + // on successful execution: + + // deduplicate the lines in each file in the store-field-dir + //use options.StoreFieldDir once https://github.com/projectdiscovery/katana/pull/877 is merged + storeFieldDir := "katana_field" + _ = deduplicateLinesInFilesInDir(storeFieldDir) + + // remove the resume file in case it exists if fileutil.FileExists(resumeFilename) { os.Remove(resumeFilename) } } +func deduplicateLinesInFilesInDir(dir string) error { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + return deduplicateLinesInFile(path) + } + return nil + }) + if err != nil { + return errorutil.NewWithErr(err).Msgf("error processing directory %s", dir) + } + return nil +} + +func deduplicateLinesInFile(filename string) error { + file, err := os.Open(filename) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not open file: %s", filename) + } + defer file.Close() + + seenLines := make(map[string]struct{}) + var deduplicatedLines []string + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if _, exists := seenLines[line]; !exists { + seenLines[line] = struct{}{} + deduplicatedLines = append(deduplicatedLines, line) + } + } + + if err := scanner.Err(); err != nil { + return errorutil.NewWithErr(err).Msgf("could not read file: %s", filename) + } + + return os.WriteFile(filename, []byte(strings.Join(deduplicatedLines, "\n")+"\n"), 0644) +} + func readFlags() (*goflags.FlagSet, error) { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Katana is a fast crawler focused on execution in automation From 77946c00a8fd8f9035486c1741d077baa576fbc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 10 May 2024 11:52:50 +0300 Subject: [PATCH 044/183] update utils --- cmd/katana/main.go | 46 ++-------------------------------------------- go.mod | 4 ++-- go.sum | 3 +++ 3 files changed, 7 insertions(+), 46 deletions(-) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index ff415265..09334f44 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "fmt" "math" "os" @@ -18,6 +17,7 @@ import ( "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" "github.com/rs/xid" ) @@ -74,7 +74,7 @@ func main() { // deduplicate the lines in each file in the store-field-dir //use options.StoreFieldDir once https://github.com/projectdiscovery/katana/pull/877 is merged storeFieldDir := "katana_field" - _ = deduplicateLinesInFilesInDir(storeFieldDir) + _ = folderutil.DedupeLinesInFiles(storeFieldDir) // remove the resume file in case it exists if fileutil.FileExists(resumeFilename) { @@ -83,48 +83,6 @@ func main() { } -func deduplicateLinesInFilesInDir(dir string) error { - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - return deduplicateLinesInFile(path) - } - return nil - }) - if err != nil { - return errorutil.NewWithErr(err).Msgf("error processing directory %s", dir) - } - return nil -} - -func deduplicateLinesInFile(filename string) error { - file, err := os.Open(filename) - if err != nil { - return errorutil.NewWithErr(err).Msgf("could not open file: %s", filename) - } - defer file.Close() - - seenLines := make(map[string]struct{}) - var deduplicatedLines []string - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if _, exists := seenLines[line]; !exists { - seenLines[line] = struct{}{} - deduplicatedLines = append(deduplicatedLines, line) - } - } - - if err := scanner.Err(); err != nil { - return errorutil.NewWithErr(err).Msgf("could not read file: %s", filename) - } - - return os.WriteFile(filename, []byte(strings.Join(deduplicatedLines, "\n")+"\n"), 0644) -} - func readFlags() (*goflags.FlagSet, error) { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Katana is a fast crawler focused on execution in automation diff --git a/go.mod b/go.mod index 754d8b46..fd980ef3 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 - github.com/projectdiscovery/fastdialer v0.0.69 + github.com/projectdiscovery/fastdialer v0.0.70 github.com/projectdiscovery/goflags v0.1.51 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.39 github.com/projectdiscovery/retryablehttp-go v1.0.59 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.92 + github.com/projectdiscovery/utils v0.0.93 github.com/projectdiscovery/wappalyzergo v0.0.121 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index d0190144..367b0e12 100644 --- a/go.sum +++ b/go.sum @@ -210,6 +210,7 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= +github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4= github.com/projectdiscovery/goflags v0.1.51 h1:PhMekTX727L1YqBfP0of0clSygrq20RnWWp0+khAxqo= github.com/projectdiscovery/goflags v0.1.51/go.mod h1:muJxbcOFi4kzg0G30S526X6dY/OyuDInchYIkoZOudU= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -236,6 +237,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= +github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= +github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= github.com/projectdiscovery/wappalyzergo v0.0.121/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From ca9465c0eec84222fd975c003d37ba7af22e4c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 10 May 2024 11:53:47 +0300 Subject: [PATCH 045/183] go mod tidy --- go.sum | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go.sum b/go.sum index 367b0e12..4a159fe5 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,7 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= -github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= +github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U= github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4= github.com/projectdiscovery/goflags v0.1.51 h1:PhMekTX727L1YqBfP0of0clSygrq20RnWWp0+khAxqo= github.com/projectdiscovery/goflags v0.1.51/go.mod h1:muJxbcOFi4kzg0G30S526X6dY/OyuDInchYIkoZOudU= @@ -235,8 +234,6 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= -github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= From fd3d50af911e4fc1d13ba62c43d14045eca58843 Mon Sep 17 00:00:00 2001 From: "David A. Symons" Date: Sat, 11 May 2024 21:19:39 +0100 Subject: [PATCH 046/183] Updating Dockerfile to more recent golang and alpine images --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a86f46f..216083c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM golang:1.20.6-alpine AS builder +FROM golang:alpine3.19 AS builder RUN apk add --no-cache git gcc musl-dev WORKDIR /app COPY . /app RUN go mod download RUN go build ./cmd/katana -FROM alpine:3.18.5 +FROM alpine:3.19.1 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/katana /usr/local/bin/ From d31663553a82825ff53ee44b4355ddf7f51420c9 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 13 May 2024 09:42:31 +0200 Subject: [PATCH 047/183] restoring dep bot --- .github/dependabot.yml | 46 +++++++++++++++++++++--------------------- Dockerfile | 4 ++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d0b098d1..0fef44f5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,26 +20,26 @@ updates: allow: - dependency-name: "github.com/projectdiscovery/*" -# # Maintain dependencies for GitHub Actions -# - package-ecosystem: "github-actions" -# directory: "/" -# schedule: -# interval: "weekly" -# target-branch: "dev" -# commit-message: -# prefix: "chore" -# include: "scope" -# labels: -# - "Type: Maintenance" -# -# # Maintain dependencies for docker -# - package-ecosystem: "docker" -# directory: "/" -# schedule: -# interval: "weekly" -# target-branch: "dev" -# commit-message: -# prefix: "chore" -# include: "scope" -# labels: -# - "Type: Maintenance" \ No newline at end of file + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" + + # Maintain dependencies for docker + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 216083c9..2a86f46f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM golang:alpine3.19 AS builder +FROM golang:1.20.6-alpine AS builder RUN apk add --no-cache git gcc musl-dev WORKDIR /app COPY . /app RUN go mod download RUN go build ./cmd/katana -FROM alpine:3.19.1 +FROM alpine:3.18.5 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/katana /usr/local/bin/ From 71683be3f20e987c8fcceba2b8506026df560152 Mon Sep 17 00:00:00 2001 From: michael2to3 <87614587+michael2to3@users.noreply.github.com> Date: Mon, 13 May 2024 12:13:55 +0000 Subject: [PATCH 048/183] user-agent header Issue resolved in headless mode (#868) * Update README.md * fix: add user agent handler * small refactor --------- Co-authored-by: zy9ard3 <67743789+zy9ard3@users.noreply.github.com> Co-authored-by: Sandeep Singh Co-authored-by: Mzack9999 --- pkg/engine/hybrid/crawl.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index f3a7d628..c100d36d 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -237,14 +237,28 @@ func (c *Crawler) addHeadersToPage(page *rod.Page) { if len(c.Headers) == 0 { return } + var arr []string + for k, v := range c.Headers { - arr = append(arr, k, v) + switch { + case stringsutil.EqualFoldAny(k, "User-Agent"): + userAgentParams := &proto.NetworkSetUserAgentOverride{ + UserAgent: v, + } + if err := page.SetUserAgent(userAgentParams); err != nil { + gologger.Error().Msgf("headless: could not set user agent: %v", err) + } + default: + arr = append(arr, k, v) + } } - // ignore cleanup callback - _, err := page.SetExtraHeaders(arr) - if err != nil { - gologger.Error().Msgf("headless: could not set extra headers: %v", err) + + if len(arr) > 0 { + _, err := page.SetExtraHeaders(arr) + if err != nil { + gologger.Error().Msgf("headless: could not set extra headers: %v", err) + } } } From 6ba85dc693d5b6a60ce05ca7bae9bda53b8b2354 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 12:48:15 +0000 Subject: [PATCH 049/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.47 to 0.0.51. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.47...v0.0.51) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fd980ef3..899ea5fe 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.39 github.com/projectdiscovery/retryablehttp-go v1.0.59 - github.com/projectdiscovery/useragent v0.0.47 + github.com/projectdiscovery/useragent v0.0.51 github.com/projectdiscovery/utils v0.0.93 github.com/projectdiscovery/wappalyzergo v0.0.121 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 4a159fe5..a75013c9 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.59 h1:jyeBhFG4iKpJPurlQAlVLgCI github.com/projectdiscovery/retryablehttp-go v1.0.59/go.mod h1:/L7lD+gOM/44r1mWgwf3knG9Q6gUbNEtbd1DNEq1+bE= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= -github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= +github.com/projectdiscovery/useragent v0.0.51 h1:0bQqhcuqSvgQUEQgkW0LrxXqOaI1U+46oyhiW5LUJvo= +github.com/projectdiscovery/useragent v0.0.51/go.mod h1:2YM5heuHH3DaNgE2ai0XCT9U6aDwhCZAxQw3VXxkLpk= github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= From 166f83f6bd5f01d9fd04cd13d3139ec7f5854665 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 12:48:19 +0000 Subject: [PATCH 050/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.39 to 0.0.41. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.39...v0.0.41) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fd980ef3..49808635 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.39 + github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.59 github.com/projectdiscovery/useragent v0.0.47 github.com/projectdiscovery/utils v0.0.93 diff --git a/go.sum b/go.sum index 4a159fe5..30d3531d 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc= -github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0= +github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm2HFxguA8a+6E= +github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= github.com/projectdiscovery/retryablehttp-go v1.0.59 h1:jyeBhFG4iKpJPurlQAlVLgCImTMjwUBaJmZoz90SnVw= From 6b429e8079b457e04ec8ecfd5941d1fff5507e5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 12:48:23 +0000 Subject: [PATCH 051/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.70 to 0.0.71. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.70...v0.0.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fd980ef3..a28667a8 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 - github.com/projectdiscovery/fastdialer v0.0.70 + github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/goflags v0.1.51 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.41 diff --git a/go.sum b/go.sum index 4a159fe5..5189e181 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.70 h1:1rnUKc8NRj6dzG8aTmqW+RF8m0dFbfTs9CiOo6Aig8U= -github.com/projectdiscovery/fastdialer v0.0.70/go.mod h1:HQ0ZpvOPOTZFSQxGyYJgNdek93hi4eIC1avZgiQ7+a4= +github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= +github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/goflags v0.1.51 h1:PhMekTX727L1YqBfP0of0clSygrq20RnWWp0+khAxqo= github.com/projectdiscovery/goflags v0.1.51/go.mod h1:muJxbcOFi4kzg0G30S526X6dY/OyuDInchYIkoZOudU= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= From 5c63e117e33e3aa80b497a33a09e0992bc49ae63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 13:14:57 +0000 Subject: [PATCH 052/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.59 to 1.0.60. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.59...v1.0.60) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6123cfd5..3ee356a0 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 - github.com/projectdiscovery/retryablehttp-go v1.0.59 + github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/useragent v0.0.51 github.com/projectdiscovery/utils v0.0.93 github.com/projectdiscovery/wappalyzergo v0.0.121 diff --git a/go.sum b/go.sum index 630e5d18..ba2780bc 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.59 h1:jyeBhFG4iKpJPurlQAlVLgCImTMjwUBaJmZoz90SnVw= -github.com/projectdiscovery/retryablehttp-go v1.0.59/go.mod h1:/L7lD+gOM/44r1mWgwf3knG9Q6gUbNEtbd1DNEq1+bE= +github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= +github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/useragent v0.0.51 h1:0bQqhcuqSvgQUEQgkW0LrxXqOaI1U+46oyhiW5LUJvo= From 63b532f53ae171633faf85f8081ae85400f49680 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 13:41:12 +0000 Subject: [PATCH 053/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.41 to 0.0.42 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.41 to 0.0.42. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.41...v0.0.42) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3ee356a0..7ee058f1 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/goflags v0.1.51 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.41 + github.com/projectdiscovery/hmap v0.0.42 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.60 diff --git a/go.sum b/go.sum index ba2780bc..55fc72bd 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,8 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= -github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= +github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= +github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From 41779720cce1690532484fe68d1832047417eb24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 12:57:16 +0000 Subject: [PATCH 054/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.51 to 0.0.52. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.51...v0.0.52) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 5 ++--- go.sum | 11 ++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 7ee058f1..186c437c 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.52 github.com/projectdiscovery/fastdialer v0.0.71 - github.com/projectdiscovery/goflags v0.1.51 + github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.42 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.60 - github.com/projectdiscovery/useragent v0.0.51 + github.com/projectdiscovery/useragent v0.0.52 github.com/projectdiscovery/utils v0.0.93 github.com/projectdiscovery/wappalyzergo v0.0.121 github.com/remeh/sizedwaitgroup v1.0.0 @@ -71,7 +71,6 @@ require ( github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index 55fc72bd..159f19c0 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= -github.com/projectdiscovery/goflags v0.1.51 h1:PhMekTX727L1YqBfP0of0clSygrq20RnWWp0+khAxqo= -github.com/projectdiscovery/goflags v0.1.51/go.mod h1:muJxbcOFi4kzg0G30S526X6dY/OyuDInchYIkoZOudU= +github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= +github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -230,10 +230,8 @@ github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0Rs github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= -github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/useragent v0.0.51 h1:0bQqhcuqSvgQUEQgkW0LrxXqOaI1U+46oyhiW5LUJvo= -github.com/projectdiscovery/useragent v0.0.51/go.mod h1:2YM5heuHH3DaNgE2ai0XCT9U6aDwhCZAxQw3VXxkLpk= +github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= +github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= @@ -253,7 +251,6 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.14.2 h1:5DPTtR9JBjKPJS008/A409I5ntFhUPPGCmaAihcPRyo= From 785e6dcf4ed72d318f821fcdfbdf1d37f2691a7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 13:23:44 +0000 Subject: [PATCH 055/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.121 to 0.1.1. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.121...v0.1.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 186c437c..ff2f79d8 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/useragent v0.0.52 github.com/projectdiscovery/utils v0.0.93 - github.com/projectdiscovery/wappalyzergo v0.0.121 + github.com/projectdiscovery/wappalyzergo v0.1.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.24.0 + golang.org/x/net v0.25.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -94,7 +94,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.19.0 // indirect + golang.org/x/term v0.20.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) @@ -137,11 +137,11 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 159f19c0..473abf01 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08s github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= -github.com/projectdiscovery/wappalyzergo v0.0.121 h1:Xae4Yw3/pzjh1TJQdoavoV2OTo4Tn5g8J8DQQqcieDA= -github.com/projectdiscovery/wappalyzergo v0.0.121/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0= +github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= +github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -376,8 +376,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -403,8 +403,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -444,16 +444,16 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -463,8 +463,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= From 10f572b1086cfe53770301bfda4d80fd6972fabc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 13:23:44 +0000 Subject: [PATCH 056/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.52 to 0.0.57 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.52 to 0.0.57. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.52...v0.0.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 186c437c..6456b96d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.52 + github.com/projectdiscovery/dsl v0.0.57 github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 diff --git a/go.sum b/go.sum index 159f19c0..4e3edd19 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= -github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= +github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= +github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= From d71aabaf4110be6a85c7ea41e7e272f44a8720f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 13:50:35 +0000 Subject: [PATCH 057/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.71 to 0.0.72. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.71...v0.0.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6223397f..fa9e7a1c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.57 - github.com/projectdiscovery/fastdialer v0.0.71 + github.com/projectdiscovery/fastdialer v0.0.72 github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.42 diff --git a/go.sum b/go.sum index 09aeba57..5c0e4b60 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= -github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= -github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= +github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= +github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= From f4b1b15647e5953216e481183cd029533479c3c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 13:51:21 +0000 Subject: [PATCH 058/183] chore(deps): bump github.com/projectdiscovery/utils Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.93 to 0.0.94. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.93...v0.0.94) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6223397f..95fcc60a 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.60 github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.93 + github.com/projectdiscovery/utils v0.0.94 github.com/projectdiscovery/wappalyzergo v0.1.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 09aeba57..7ca8f625 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.93 h1:IMZFsmQFYZUf7rxpBoZj+53FsNDC/vHsXA+4B4GuGeg= -github.com/projectdiscovery/utils v0.0.93/go.mod h1:2+mWzk5FeYdK9imo5eLk6oVeih0G0wsTff1pzBAh9tk= +github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= +github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 24b23b35d9238beb0dc9072e3516ad19ff1c5310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 20 May 2024 23:39:45 +0300 Subject: [PATCH 059/183] remove passive crawling --- README.md | 5 - cmd/katana/main.go | 4 - internal/runner/options.go | 4 - internal/runner/runner.go | 3 - pkg/engine/common/base.go | 15 +- pkg/engine/passive/doc.go | 3 - pkg/engine/passive/httpclient/httpclient.go | 124 --------------- pkg/engine/passive/passive.go | 145 ------------------ pkg/engine/passive/regexp/regexp.go | 16 -- pkg/engine/passive/registry.go | 14 -- .../passive/source/alienvault/alienvault.go | 81 ---------- .../passive/source/commoncrawl/commoncrawl.go | 136 ---------------- pkg/engine/passive/source/source.go | 21 --- .../source/waybackarchive/waybackarchive.go | 65 -------- pkg/navigation/response.go | 5 - pkg/output/result.go | 9 +- pkg/types/options.go | 4 - 17 files changed, 11 insertions(+), 643 deletions(-) delete mode 100644 pkg/engine/passive/doc.go delete mode 100644 pkg/engine/passive/httpclient/httpclient.go delete mode 100644 pkg/engine/passive/passive.go delete mode 100644 pkg/engine/passive/regexp/regexp.go delete mode 100644 pkg/engine/passive/registry.go delete mode 100644 pkg/engine/passive/source/alienvault/alienvault.go delete mode 100644 pkg/engine/passive/source/commoncrawl/commoncrawl.go delete mode 100644 pkg/engine/passive/source/source.go delete mode 100644 pkg/engine/passive/source/waybackarchive/waybackarchive.go diff --git a/README.md b/README.md index 2011262e..ab119f1f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ - Fast And fully configurable web crawling - **Standard** and **Headless** mode - - **Active** and **Passive** mode - **JavaScript** parsing / crawling - Customizable **automatic form filling** - **Scope control** - Preconfigured field / Regex @@ -157,10 +156,6 @@ HEADLESS: -cwu, -chrome-ws-url string use chrome browser instance launched elsewhere with the debugger listening at this URL -xhr, -xhr-extraction extract xhr request url,method in jsonl output -PASSIVE: - -ps, -passive enable passive sources to discover target endpoints - -pss, -passive-source string[] passive source to use for url discovery (waybackarchive,commoncrawl,alienvault) - SCOPE: -cs, -crawl-scope string[] in scope url regex to be followed by crawler -cos, -crawl-out-scope string[] out of scope url regex to be excluded by crawler diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 09334f44..a7336296 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -134,10 +134,6 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.ChromeWSUrl, "chrome-ws-url", "cwu", "", "use chrome browser instance launched elsewhere with the debugger listening at this URL"), flagSet.BoolVarP(&options.XhrExtraction, "xhr-extraction", "xhr", false, "extract xhr request url,method in jsonl output"), ) - flagSet.CreateGroup("passive", "Passive", - flagSet.BoolVarP(&options.Passive, "passive", "ps", false, "enable passive sources to discover target endpoints"), - flagSet.StringSliceVarP(&options.PassiveSource, "passive-source", "pss", nil, "passive source to use for url discovery (waybackarchive,commoncrawl,alienvault)", goflags.NormalizedStringSliceOptions), - ) flagSet.CreateGroup("scope", "Scope", flagSet.StringSliceVarP(&options.Scope, "crawl-scope", "cs", nil, "in scope url regex to be followed by crawler", goflags.FileCommaSeparatedStringSliceOptions), diff --git a/internal/runner/options.go b/internal/runner/options.go index 0c7fd0fb..5bb847fe 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,10 +25,6 @@ func validateOptions(options *types.Options) error { return errorutil.New("no inputs specified for crawler") } - if options.Headless && options.Passive { - return errorutil.New("headless mode (-headless) and passive mode (-passive) cannot be used together") - } - if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { return errorutil.New("headless mode (-hl) is required if -ho, -nos or -scp are set") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 03d9118e..7aacf1f8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -9,7 +9,6 @@ import ( "github.com/projectdiscovery/katana/pkg/engine" "github.com/projectdiscovery/katana/pkg/engine/hybrid" "github.com/projectdiscovery/katana/pkg/engine/parser" - "github.com/projectdiscovery/katana/pkg/engine/passive" "github.com/projectdiscovery/katana/pkg/engine/standard" "github.com/projectdiscovery/katana/pkg/types" "github.com/projectdiscovery/mapcidr" @@ -98,8 +97,6 @@ func New(options *types.Options) (*Runner, error) { switch { case options.Headless: crawler, err = hybrid.New(crawlerOptions) - case options.Passive: - crawler, err = passive.New(crawlerOptions) default: crawler, err = standard.New(crawlerOptions) } diff --git a/pkg/engine/common/base.go b/pkg/engine/common/base.go index 76bbf8cc..7eec2a49 100644 --- a/pkg/engine/common/base.go +++ b/pkg/engine/common/base.go @@ -72,7 +72,7 @@ func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.R // if the user requested anyway out of scope items // they are sent to output without visiting if s.Options.Options.DisplayOutScope { - s.Output(nr, nil, nil, ErrOutOfScope) + s.Output(nr, nil, ErrOutOfScope) } continue } @@ -95,18 +95,17 @@ func (s *Shared) ValidateScope(URL string, root string) bool { return err == nil && scopeValidated } -func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, passiveReference *navigation.PassiveReference, err error) { +func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, err error) { var errData string if err != nil { errData = err.Error() } // Write the found result to output result := &output.Result{ - Timestamp: time.Now(), - Request: navigationRequest, - Response: navigationResponse, - PassiveReference: passiveReference, - Error: errData, + Timestamp: time.Now(), + Request: navigationRequest, + Response: navigationResponse, + Error: errData, } outputErr := s.Options.OutputWriter.Write(result) @@ -224,7 +223,7 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { resp, err := doRequest(crawlSession, req) - s.Output(req, resp, nil, err) + s.Output(req, resp, err) if err != nil { gologger.Warning().Msgf("Could not request seed URL %s: %s\n", req.URL, err) diff --git a/pkg/engine/passive/doc.go b/pkg/engine/passive/doc.go deleted file mode 100644 index 915dd7b4..00000000 --- a/pkg/engine/passive/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package passive implements the functionality for a non-headless crawler. -// It uses net/http for making requests and goquery for scraping web page HTML. -package passive diff --git a/pkg/engine/passive/httpclient/httpclient.go b/pkg/engine/passive/httpclient/httpclient.go deleted file mode 100644 index e3c671da..00000000 --- a/pkg/engine/passive/httpclient/httpclient.go +++ /dev/null @@ -1,124 +0,0 @@ -package httpclient - -import ( - "bytes" - "context" - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "net/url" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/useragent" -) - -type HttpClient struct { - Client *http.Client -} - -type BasicAuth struct { - Username string - Password string -} - -func NewHttpClient(timeout int) *HttpClient { - Transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - Dial: (&net.Dialer{ - Timeout: time.Duration(timeout) * time.Second, - }).Dial, - } - - client := &http.Client{ - Transport: Transport, - Timeout: time.Duration(timeout) * time.Second, - } - - httpClient := &HttpClient{Client: client} - - return httpClient -} - -func (hc *HttpClient) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { - return hc.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) -} - -func (hc *HttpClient) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { - return hc.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) -} - -func (hc *HttpClient) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { - return hc.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) -} - -func (hc *HttpClient) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { - return hc.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) -} - -func (hc *HttpClient) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, method, requestURL, body) - if err != nil { - return nil, err - } - - userAgent := useragent.PickRandom() - req.Header.Set("User-Agent", userAgent.String()) - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") - req.Header.Set("Connection", "close") - - if basicAuth.Username != "" || basicAuth.Password != "" { - req.SetBasicAuth(basicAuth.Username, basicAuth.Password) - } - - if cookies != "" { - req.Header.Set("Cookie", cookies) - } - - for key, value := range headers { - req.Header.Set(key, value) - } - - return httpRequestWrapper(hc.Client, req) -} - -func (hc *HttpClient) DiscardHTTPResponse(response *http.Response) { - if response != nil { - _, err := io.Copy(io.Discard, response.Body) - if err != nil { - gologger.Warning().Msgf("Could not discard response body: %s\n", err) - return - } - response.Body.Close() - } -} - -func (hc *HttpClient) Close() { - hc.Client.CloseIdleConnections() -} - -func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { - response, err := client.Do(request) - if err != nil { - return nil, err - } - - if response.StatusCode != http.StatusOK { - requestURL, _ := url.QueryUnescape(request.URL.String()) - - gologger.Debug().MsgFunc(func() string { - buffer := new(bytes.Buffer) - _, _ = buffer.ReadFrom(response.Body) - return fmt.Sprintf("Response for failed request against %s:\n%s", requestURL, buffer.String()) - }) - return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL) - } - return response, nil -} diff --git a/pkg/engine/passive/passive.go b/pkg/engine/passive/passive.go deleted file mode 100644 index 0a6188f7..00000000 --- a/pkg/engine/passive/passive.go +++ /dev/null @@ -1,145 +0,0 @@ -package passive - -import ( - "context" - "fmt" - "net/http" - "strings" - "sync" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/katana/pkg/engine/common" - "github.com/projectdiscovery/katana/pkg/engine/passive/httpclient" - "github.com/projectdiscovery/katana/pkg/engine/passive/source" - "github.com/projectdiscovery/katana/pkg/navigation" - "github.com/projectdiscovery/katana/pkg/types" - "github.com/projectdiscovery/katana/pkg/utils" - errorutil "github.com/projectdiscovery/utils/errors" - urlutil "github.com/projectdiscovery/utils/url" - "golang.org/x/exp/maps" -) - -// Crawler is a passive crawler instance -type Crawler struct { - *common.Shared - sources []source.Source - httpClient *httpclient.HttpClient -} - -// New returns a new passive crawler instance -func New(options *types.CrawlerOptions) (*Crawler, error) { - shared, err := common.NewShared(options) - if err != nil { - return nil, errorutil.NewWithErr(err).WithTag("passive") - } - - sources := make(map[string]source.Source, len(Sources)) - if len(options.Options.PassiveSource) > 0 { - for _, source := range options.Options.PassiveSource { - if s, ok := Sources[source]; ok { - sources[source] = s - } - } - } else { - sources = Sources - } - - if len(sources) == 0 { - gologger.Fatal().Msg("No sources selected for this search") - } - - gologger.Debug().Msgf(fmt.Sprintf("Selected source(s) for this crawl: %s", strings.Join(maps.Keys(sources), ", "))) - - httpClient := httpclient.NewHttpClient(options.Options.Timeout) - return &Crawler{Shared: shared, sources: maps.Values(sources), httpClient: httpClient}, nil -} - -// Close closes the crawler process -func (c *Crawler) Close() error { - return nil -} - -// Crawl crawls a URL with the specified options -func (c *Crawler) Crawl(rootURL string) error { - gologger.Info().Msgf("Enumerating passive endpoints for %s", rootURL) - - rootUrlParsed, _ := urlutil.ParseURL(rootURL, true) - results := make(chan source.Result) - var timeTaken time.Duration - go func() { - defer func(startTime time.Time) { - timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - ctx := context.Background() - wg := &sync.WaitGroup{} - for _, s := range c.sources { - wg.Add(1) - go func(source source.Source) { - for result := range source.Run(ctx, c.Shared, rootURL) { - results <- result - } - wg.Done() - }(s) - } - wg.Wait() - }() - - seenURLs := make(map[string]struct{}) - sourceStats := make(map[string]int) - for result := range results { - if _, found := seenURLs[result.Value]; found { - continue - } - - if !utils.IsURL(result.Value) { - gologger.Debug().Msgf("`%v` not a url. skipping", result.Value) - continue - } - - if ok, err := c.Options.ValidateScope(result.Value, rootUrlParsed.Hostname()); err != nil || !ok { - gologger.Debug().Msgf("`%v` not in scope. skipping", result.Value) - continue - } - - if !c.Options.ExtensionsValidator.ValidatePath(result.Value) { - gologger.Debug().Msgf("`%v` not allowed extension. skipping", result.Value) - continue - } - - seenURLs[result.Value] = struct{}{} - sourceStats[result.Source]++ - - passiveURL, _ := urlutil.Parse(result.Value) - req := &navigation.Request{ - Method: http.MethodGet, - URL: result.Value, - } - resp := &navigation.Response{ - StatusCode: http.StatusOK, - RootHostname: passiveURL.Hostname(), - Resp: &http.Response{ - StatusCode: http.StatusOK, - Request: &http.Request{ - Method: http.MethodGet, - URL: passiveURL.URL, - }, - }, - } - passiveReference := &navigation.PassiveReference{ - Source: result.Source, - Reference: result.Reference, - } - c.Output(req, resp, passiveReference, nil) - } - - var stats []string - for source, count := range sourceStats { - stats = append(stats, fmt.Sprintf("%s: %d", source, count)) - } - - gologger.Info().Msgf("Found %d endpoints for %s in %s (%s)", len(seenURLs), rootURL, timeTaken.String(), strings.Join(stats, ", ")) - return nil -} diff --git a/pkg/engine/passive/regexp/regexp.go b/pkg/engine/passive/regexp/regexp.go deleted file mode 100644 index 785d7f90..00000000 --- a/pkg/engine/passive/regexp/regexp.go +++ /dev/null @@ -1,16 +0,0 @@ -package regexp - -import ( - "regexp" - "strings" -) - -var re, _ = regexp.Compile(`(?:http|https)?://(?:www\.)?[a-zA-Z0-9./?=_%:-]*`) - -func Extract(text string) []string { - matches := re.FindAllString(text, -1) - for i, match := range matches { - matches[i] = strings.ToLower(match) - } - return matches -} diff --git a/pkg/engine/passive/registry.go b/pkg/engine/passive/registry.go deleted file mode 100644 index b3a7f1d6..00000000 --- a/pkg/engine/passive/registry.go +++ /dev/null @@ -1,14 +0,0 @@ -package passive - -import ( - "github.com/projectdiscovery/katana/pkg/engine/passive/source" - "github.com/projectdiscovery/katana/pkg/engine/passive/source/alienvault" - "github.com/projectdiscovery/katana/pkg/engine/passive/source/commoncrawl" - "github.com/projectdiscovery/katana/pkg/engine/passive/source/waybackarchive" -) - -var Sources = map[string]source.Source{ - "waybackarchive": &waybackarchive.Source{}, - "commoncrawl": &commoncrawl.Source{}, - "alienvault": &alienvault.Source{}, -} diff --git a/pkg/engine/passive/source/alienvault/alienvault.go b/pkg/engine/passive/source/alienvault/alienvault.go deleted file mode 100644 index 8a4e25d3..00000000 --- a/pkg/engine/passive/source/alienvault/alienvault.go +++ /dev/null @@ -1,81 +0,0 @@ -package alienvault - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/projectdiscovery/katana/pkg/engine/common" - "github.com/projectdiscovery/katana/pkg/engine/passive/httpclient" - "github.com/projectdiscovery/katana/pkg/engine/passive/source" - urlutil "github.com/projectdiscovery/utils/url" -) - -type alienvaultResponse struct { - URLList []url `json:"url_list"` - HasNext bool `json:"has_next"` -} - -type url struct { - URL string `json:"url"` -} - -type Source struct { -} - -func (s *Source) Run(ctx context.Context, sharedCtx *common.Shared, rootUrl string) <-chan source.Result { - results := make(chan source.Result) - - go func() { - defer close(results) - - if parsedRootUrl, err := urlutil.Parse(rootUrl); err == nil { - rootUrl = parsedRootUrl.Hostname() - } - - httpClient := httpclient.NewHttpClient(sharedCtx.Options.Options.Timeout) - page := 1 - for { - apiURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?page=%d", rootUrl, page) - resp, err := httpClient.SimpleGet(ctx, apiURL) - if err != nil && resp == nil { - results <- source.Result{Source: s.Name(), Error: err} - httpClient.DiscardHTTPResponse(resp) - return - } - - var response alienvaultResponse - // Get the response body and decode - err = json.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- source.Result{Source: s.Name(), Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - for _, record := range response.URLList { - results <- source.Result{Source: s.Name(), Value: record.URL, Reference: apiURL} - } - - if !response.HasNext { - break - } - page++ - } - }() - - return results -} - -func (s *Source) Name() string { - return "alienvault" -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} diff --git a/pkg/engine/passive/source/commoncrawl/commoncrawl.go b/pkg/engine/passive/source/commoncrawl/commoncrawl.go deleted file mode 100644 index 9ea8a8dc..00000000 --- a/pkg/engine/passive/source/commoncrawl/commoncrawl.go +++ /dev/null @@ -1,136 +0,0 @@ -// Package commoncrawl logic -package commoncrawl - -import ( - "bufio" - "context" - "fmt" - "net/url" - "strconv" - "strings" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/katana/pkg/engine/common" - "github.com/projectdiscovery/katana/pkg/engine/passive/httpclient" - "github.com/projectdiscovery/katana/pkg/engine/passive/regexp" - "github.com/projectdiscovery/katana/pkg/engine/passive/source" -) - -const ( - indexURL = "https://index.commoncrawl.org/collinfo.json" - maxYearsBack = 5 -) - -var year = time.Now().Year() - -type indexResponse struct { - ID string `json:"id"` - APIURL string `json:"cdx-api"` -} - -type Source struct { -} - -func (s *Source) Run(ctx context.Context, sharedCtx *common.Shared, rootUrl string) <-chan source.Result { - results := make(chan source.Result) - - go func() { - defer close(results) - - httpClient := httpclient.NewHttpClient(sharedCtx.Options.Options.Timeout) - resp, err := httpClient.SimpleGet(ctx, indexURL) - if err != nil { - results <- source.Result{Source: s.Name(), Error: err} - httpClient.DiscardHTTPResponse(resp) - return - } - - var indexes []indexResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) - if err != nil { - results <- source.Result{Source: s.Name(), Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - years := make([]string, 0) - for i := 0; i < maxYearsBack; i++ { - years = append(years, strconv.Itoa(year-i)) - } - - searchIndexes := make(map[string]string) - for _, year := range years { - for _, index := range indexes { - if strings.Contains(index.ID, year) { - if _, ok := searchIndexes[year]; !ok { - searchIndexes[year] = index.APIURL - break - } - } - } - } - - for _, apiURL := range searchIndexes { - further := s.getURLs(ctx, apiURL, rootUrl, httpClient, results) - if !further { - break - } - } - }() - - return results -} - -func (s *Source) Name() string { - return "commoncrawl" -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} - -func (s *Source) getURLs(ctx context.Context, searchURL, rootURL string, httpClient *httpclient.HttpClient, results chan source.Result) bool { - for { - select { - case <-ctx.Done(): - return false - default: - var headers = map[string]string{"Host": "index.commoncrawl.org"} - currentSearchURL := fmt.Sprintf("%s?url=*.%s", searchURL, rootURL) - resp, err := httpClient.Get(ctx, currentSearchURL, "", headers) - if err != nil { - results <- source.Result{Source: s.Name(), Error: err} - httpClient.DiscardHTTPResponse(resp) - return false - } - - scanner := bufio.NewScanner(resp.Body) - - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - line, _ = url.QueryUnescape(line) - for _, extractedURL := range regexp.Extract(line) { - // fix for triple encoded URL - extractedURL = strings.ToLower(extractedURL) - extractedURL = strings.TrimPrefix(extractedURL, "25") - extractedURL = strings.TrimPrefix(extractedURL, "2f") - if extractedURL != "" { - results <- source.Result{Source: s.Name(), Value: extractedURL, Reference: currentSearchURL} - } - } - } - resp.Body.Close() - return true - } - } -} diff --git a/pkg/engine/passive/source/source.go b/pkg/engine/passive/source/source.go deleted file mode 100644 index db5bc897..00000000 --- a/pkg/engine/passive/source/source.go +++ /dev/null @@ -1,21 +0,0 @@ -package source - -import ( - "context" - - "github.com/projectdiscovery/katana/pkg/engine/common" -) - -type Source interface { - Run(context.Context, *common.Shared, string) <-chan Result - Name() string - NeedsKey() bool - AddApiKeys([]string) -} - -type Result struct { - Source string - Value string - Reference string - Error error -} diff --git a/pkg/engine/passive/source/waybackarchive/waybackarchive.go b/pkg/engine/passive/source/waybackarchive/waybackarchive.go deleted file mode 100644 index db023d75..00000000 --- a/pkg/engine/passive/source/waybackarchive/waybackarchive.go +++ /dev/null @@ -1,65 +0,0 @@ -package waybackarchive - -import ( - "bufio" - "context" - "fmt" - "net/url" - "strings" - - "github.com/projectdiscovery/katana/pkg/engine/common" - "github.com/projectdiscovery/katana/pkg/engine/passive/httpclient" - "github.com/projectdiscovery/katana/pkg/engine/passive/regexp" - "github.com/projectdiscovery/katana/pkg/engine/passive/source" -) - -type Source struct { -} - -func (s *Source) Run(ctx context.Context, sharedCtx *common.Shared, rootUrl string) <-chan source.Result { - results := make(chan source.Result) - go func() { - defer close(results) - - httpClient := httpclient.NewHttpClient(sharedCtx.Options.Options.Timeout) - searchURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", rootUrl) - resp, err := httpClient.Get(ctx, searchURL, "", nil) - if err != nil { - results <- source.Result{Source: s.Name(), Error: err} - return - } - defer resp.Body.Close() - - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - line, _ = url.QueryUnescape(line) - for _, extractedURL := range regexp.Extract(line) { - // fix for triple encoded URL - extractedURL = strings.ToLower(extractedURL) - extractedURL = strings.TrimPrefix(extractedURL, "25") - extractedURL = strings.TrimPrefix(extractedURL, "2f") - - results <- source.Result{Source: s.Name(), Value: extractedURL, Reference: searchURL} - } - - } - }() - - return results -} - -func (s *Source) Name() string { - return "waybackarchive" -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} diff --git a/pkg/navigation/response.go b/pkg/navigation/response.go index a9dd2b7e..cb6f0ef2 100644 --- a/pkg/navigation/response.go +++ b/pkg/navigation/response.go @@ -42,11 +42,6 @@ type Response struct { StoredResponsePath string `json:"stored_response_path,omitempty"` } -type PassiveReference struct { - Source string `json:"source"` - Reference string `json:"reference"` -} - func (n Response) AbsoluteURL(path string) string { if strings.HasPrefix(path, "#") { return "" diff --git a/pkg/output/result.go b/pkg/output/result.go index 90d04c3b..9c93922d 100644 --- a/pkg/output/result.go +++ b/pkg/output/result.go @@ -8,11 +8,10 @@ import ( // Result of the crawling type Result struct { - Timestamp time.Time `json:"timestamp,omitempty"` - Request *navigation.Request `json:"request,omitempty"` - Response *navigation.Response `json:"response,omitempty"` - PassiveReference *navigation.PassiveReference `json:"passive,omitempty"` - Error string `json:"error,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty"` + Request *navigation.Request `json:"request,omitempty"` + Response *navigation.Response `json:"response,omitempty"` + Error string `json:"error,omitempty"` } // HasResponse checks if the result has a valid response diff --git a/pkg/types/options.go b/pkg/types/options.go index af6bc6fb..1963830b 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -124,10 +124,6 @@ type Options struct { HeadlessNoIncognito bool // XhrExtraction extract xhr requests XhrExtraction bool - // Passive enables passive crawling - Passive bool - // PassiveSource is the list of sources for passive crawling - PassiveSource goflags.StringSlice // HealthCheck determines if a self-healthcheck should be performed HealthCheck bool // ErrorLogFile specifies a file to write with the errors of all requests From d36da0a8c0dc16d56513a53f80affa95f143601e Mon Sep 17 00:00:00 2001 From: geeknik <466878+geeknik@users.noreply.github.com> Date: Sat, 25 May 2024 02:35:00 +0000 Subject: [PATCH 060/183] Update regex.go --- pkg/utils/regex.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/utils/regex.go b/pkg/utils/regex.go index 61ead03f..4c2db7c2 100644 --- a/pkg/utils/regex.go +++ b/pkg/utils/regex.go @@ -7,10 +7,10 @@ import ( var ( BodyA0 = `(?:` BodyB0 = `(` - BodyC0 = `(?:[\.]{1,2}/[A-Za-z0-9\-_/\\?&@\.?=%]+)` - BodyC1 = `|(https?://[A-Za-z0-9_\-\.]+([\.]{0,2})?\/[A-Za-z0-9\-_/\\?&@\.?=%]+)` - BodyC2 = `|(/[A-Za-z0-9\-_/\\?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` - BodyC3 = `|([A-Za-z0-9\-_?&@\.%]+/[A-Za-z0-9/\\\-_?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` + BodyC0 = `(?:\.\./[A-Za-z0-9\-_/\\?&@.=%]+)` + BodyC1 = `|(https?://[A-Za-z0-9_\-.]+(?:\.\./)?/[A-Za-z0-9\-_/\\?&@.=%]+)` + BodyC2 = `|(/[A-Za-z0-9\-_/\\?&@.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(?:p|on)?|pdf|php5?|py|rss))` + BodyC3 = `|([A-Za-z0-9\-_?&@.%]+/[A-Za-z0-9/\\\-_?&@.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(?:p|on)?|pdf|php5?|py|rss))` BodyB1 = `)` BodyA1 = `)` // pageBodyRegex extracts endpoints from page body @@ -18,10 +18,10 @@ var ( JsA0 = `(?:"|'|\s)` JsB0 = `(` - JsC0 = `((https?://[A-Za-z0-9_\-\.]+(:\d{1,5})?)+([\.]{1,2})?/[A-Za-z0-9/\-_\.\\%]+([\?|#][^"']+)?)` - JsC1 = `|((\.{1,2}/)?[a-zA-Z0-9\-_/\\%]+\.(aspx?|js(on|p)?|html|php5?|html|action|do)([\?|#][^"']+)?)` - JsC2 = `|((\.{0,2}/)[a-zA-Z0-9\-_/\\%]+(/|\\)[a-zA-Z0-9\-_]{3,}([\?|#][^"|']+)?)` - JsC3 = `|((\.{0,2})[a-zA-Z0-9\-_/\\%]{3,}/)` + JsC0 = `((https?://[A-Za-z0-9_\-.]+(?:\:\d{1,5})?)+(?:\.\./)?/[A-Za-z0-9/\-_\\.%]+(?:[\?|#][^"']+)?)` + JsC1 = `|((?:\.\./)?[a-zA-Z0-9\-_/\\%]+\.(aspx?|js(?:on|p)?|html|php5?|action|do)(?:[\?|#][^"']+)?)` + JsC2 = `|((?:\.\./)[a-zA-Z0-9\-_/\\%]+(?:/|\\)[a-zA-Z0-9\-_]{3,}(?:[\?|#][^"']+)?)` + JsC3 = `|((?:\.\./)[a-zA-Z0-9\-_/\\%]{3,}/)` JsB1 = `)` JsA1 = `(?:"|'|\s)` // relativeEndpointsRegex is the regex to find endpoints in js files. From 76c090da193dcc25d69cd9d8d270341ca51154d2 Mon Sep 17 00:00:00 2001 From: geeknik <466878+geeknik@users.noreply.github.com> Date: Sat, 25 May 2024 02:35:31 +0000 Subject: [PATCH 061/183] Update utils.go --- pkg/utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 80c2d90c..6c51d14e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -65,7 +65,7 @@ func ParseRefreshTag(value string) string { // WebUserAgent returns the chrome-web user agent func WebUserAgent() string { - return "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" + return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" } func FlattenHeaders(headers map[string][]string) map[string]string { From 181984dc7ea394bd4dfb3cb08b7f9c9cb680415a Mon Sep 17 00:00:00 2001 From: geeknik <466878+geeknik@users.noreply.github.com> Date: Sat, 25 May 2024 02:37:41 +0000 Subject: [PATCH 062/183] Update jsluice.go --- pkg/utils/jsluice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/jsluice.go b/pkg/utils/jsluice.go index 38e5c7b8..ffed37ca 100644 --- a/pkg/utils/jsluice.go +++ b/pkg/utils/jsluice.go @@ -10,7 +10,7 @@ import ( var ( // CommonJSLibraryFileRegex is a regex to match common js library files. - CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck)(?:[-._][\w\d]*)*\.js$` + CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck|chart[-_.]?js|plotly|echarts|d3[-_.]?force|sigma|c3|nvd3|amcharts|vis[-_.]?js|dagre[-_.]?d3|cytoscape|leaflet|openlayers|ol3|mapbox|cesium|turf|moment[-_.]?timezone|luxon|dayjs|date[-_.]?fns|date[-_.]?io|flatpickr|pikaday|fullcalendar|draggable|interact|sortable|dragula|dropzone|filepond|uppy|fine[-_.]?uploader|plyr|mediaelement|flowplayer|jwplayer|video[-_.]?js|mediaelement[-_.]?js|dash[-_.]?js|hls[-_.]?js|videojs|wavesurfer|soundmanager|amplitude|pizzicato|tone)(?:[-._][\w\d]*)*\.js$` commonJSLibraryFileRegexCompiled = regexp.MustCompile(CommonJSLibraryFileRegex) ) From 3682bd62549786d4038de5f723007793a3469c54 Mon Sep 17 00:00:00 2001 From: geeknik <466878+geeknik@users.noreply.github.com> Date: Sat, 25 May 2024 09:18:58 -0500 Subject: [PATCH 063/183] Update utils.go fix spacing --- pkg/utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 6c51d14e..3b355a45 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -65,7 +65,7 @@ func ParseRefreshTag(value string) string { // WebUserAgent returns the chrome-web user agent func WebUserAgent() string { - return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" + return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" } func FlattenHeaders(headers map[string][]string) map[string]string { From 95db66ba5c01ba1b043c9156338d5bf425f4c80f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 12:53:19 +0000 Subject: [PATCH 064/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.72 to 0.1.0. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.72...v0.1.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3da6d1b6..765babb1 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.57 - github.com/projectdiscovery/fastdialer v0.0.72 + github.com/projectdiscovery/fastdialer v0.1.0 github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.42 + github.com/projectdiscovery/hmap v0.0.43 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 - github.com/projectdiscovery/retryablehttp-go v1.0.60 + github.com/projectdiscovery/retryablehttp-go v1.0.61 github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.94 + github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -121,7 +121,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.59 // indirect + github.com/projectdiscovery/retryabledns v1.0.60 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index bd6cb2dd..7fe7351e 100644 --- a/go.sum +++ b/go.sum @@ -208,16 +208,16 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= -github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= -github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= +github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= +github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= -github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= +github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg= +github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= @@ -226,14 +226,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm2HFxguA8a+6E= github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= -github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= -github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= -github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= +github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ= +github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= +github.com/projectdiscovery/retryablehttp-go v1.0.61 h1:+LCmSWOAXd7p+S889TWhw1Fsjjdv58Ja3xL9gBUGWg8= +github.com/projectdiscovery/retryablehttp-go v1.0.61/go.mod h1:DGe6bUyvmhoMGifHA2SplUP8f8Bu87FCqZWADxj/YN8= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= -github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= +github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= +github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 04b59363093b947417140f07d804d57cef602247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 13:22:14 +0000 Subject: [PATCH 065/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.59 to 1.0.61. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.59...v1.0.61) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 765babb1..f41276a3 100644 --- a/go.mod +++ b/go.mod @@ -121,7 +121,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.60 // indirect + github.com/projectdiscovery/retryabledns v1.0.61 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 7fe7351e..2ef00cce 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm2HFxguA8a+6E= github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= -github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ= -github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= +github.com/projectdiscovery/retryabledns v1.0.61 h1:1WSX1wMZoI5IS6qdP1tgemLYtSIA7z98r1KnWY9PpbA= +github.com/projectdiscovery/retryabledns v1.0.61/go.mod h1:XnGuCyuGaR3xCOsFCY/P6O9KMT/+rnN227OAkR8QQ+4= github.com/projectdiscovery/retryablehttp-go v1.0.61 h1:+LCmSWOAXd7p+S889TWhw1Fsjjdv58Ja3xL9gBUGWg8= github.com/projectdiscovery/retryablehttp-go v1.0.61/go.mod h1:DGe6bUyvmhoMGifHA2SplUP8f8Bu87FCqZWADxj/YN8= github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= From 874274a1f7662ae6ef7684db740e96539bdfe753 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 13:22:16 +0000 Subject: [PATCH 066/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.1 to 0.1.2. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.1...v0.1.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 765babb1..af3416e9 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.61 github.com/projectdiscovery/useragent v0.0.52 github.com/projectdiscovery/utils v0.1.0 - github.com/projectdiscovery/wappalyzergo v0.1.1 + github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 7fe7351e..bfa509a0 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08s github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= -github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= -github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= +github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= +github.com/projectdiscovery/wappalyzergo v0.1.2/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From 4afe68cf564993c6e8435e44c3d02d6a2c3c5b46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 13:49:31 +0000 Subject: [PATCH 067/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.52 to 0.0.53. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.52...v0.0.53) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index af3416e9..ab990e53 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.0.57 github.com/projectdiscovery/fastdialer v0.1.0 - github.com/projectdiscovery/goflags v0.1.52 + github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.43 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.61 - github.com/projectdiscovery/useragent v0.0.52 + github.com/projectdiscovery/useragent v0.0.53 github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index bfa509a0..f4b35da3 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= -github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= -github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= +github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= +github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -230,8 +230,8 @@ github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+Un github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU= github.com/projectdiscovery/retryablehttp-go v1.0.61 h1:+LCmSWOAXd7p+S889TWhw1Fsjjdv58Ja3xL9gBUGWg8= github.com/projectdiscovery/retryablehttp-go v1.0.61/go.mod h1:DGe6bUyvmhoMGifHA2SplUP8f8Bu87FCqZWADxj/YN8= -github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= -github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= +github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= +github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= From 368034ca00c337da35484dccc18dbcfaf970706a Mon Sep 17 00:00:00 2001 From: geeknik <466878+geeknik@users.noreply.github.com> Date: Fri, 31 May 2024 15:45:00 -0500 Subject: [PATCH 068/183] Update jsluice.go ### Added - Included additional ad network libraries to the regex pattern for enhanced detection capabilities. ### Changed - Updated the `CommonJSLibraryFileRegex` to expand the coverage of JavaScript library detection by incorporating ad network libraries such as `adroll`, `doubleclick`, `facebook-pixel`, `ga-audiences`, `googlesyndication`, `adsbygoogle`, `gpt`, `amazon-adsystem`, `criteo`, `taboola`, `outbrain`, and many more. ### Fixed - Improved the accuracy and comprehensiveness of the regex pattern to ensure it captures a broader range of JavaScript library files used by both common JS frameworks and ad networks. --- pkg/utils/jsluice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/jsluice.go b/pkg/utils/jsluice.go index ffed37ca..3314956d 100644 --- a/pkg/utils/jsluice.go +++ b/pkg/utils/jsluice.go @@ -10,7 +10,7 @@ import ( var ( // CommonJSLibraryFileRegex is a regex to match common js library files. - CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck|chart[-_.]?js|plotly|echarts|d3[-_.]?force|sigma|c3|nvd3|amcharts|vis[-_.]?js|dagre[-_.]?d3|cytoscape|leaflet|openlayers|ol3|mapbox|cesium|turf|moment[-_.]?timezone|luxon|dayjs|date[-_.]?fns|date[-_.]?io|flatpickr|pikaday|fullcalendar|draggable|interact|sortable|dragula|dropzone|filepond|uppy|fine[-_.]?uploader|plyr|mediaelement|flowplayer|jwplayer|video[-_.]?js|mediaelement[-_.]?js|dash[-_.]?js|hls[-_.]?js|videojs|wavesurfer|soundmanager|amplitude|pizzicato|tone)(?:[-._][\w\d]*)*\.js$` + CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck|chart[-_.]?js|plotly|echarts|d3[-_.]?force|sigma|c3|nvd3|amcharts|vis[-_.]?js|dagre[-_.]?d3|cytoscape|leaflet|openlayers|ol3|mapbox|cesium|turf|moment[-_.]?timezone|luxon|dayjs|date[-_.]?fns|date[-_.]?io|flatpickr|pikaday|fullcalendar|draggable|interact|sortable|dragula|dropzone|filepond|uppy|fine[-_.]?uploader|plyr|mediaelement|flowplayer|jwplayer|video[-_.]?js|mediaelement[-_.]?js|dash[-_.]?js|hls[-_.]?js|videojs|wavesurfer|soundmanager|amplitude|pizzicato|tone|adroll|doubleclick|facebook-pixel|ga-audiences|googlesyndication|adsbygoogle|gpt|amazon-adsystem|criteo|taboola|outbrain|bidswitch|bidswitch.net|spotxchange|yahoo|media.net|contextweb|openx|pubmatic|rubiconproject|indexexchange|appnexus|liveintent|triplelift|verizonmedia|synacor|sonobi|yieldmo|gumgum|smartadserver|mopub|pubnative|inmobi|chartboost|tapjoy|admob|unityads|vungle|flurry|matomy|altitude|dataxu|thetradedesk|exponential|zypmedia|quantcast|mediamath|bidswitch|mgid|revcontent|powerlinks|rhythmone|airpush|smaato|adcolony|mopub|leadbolt|mobfox|nativo|revjet|smartyads|avocarrot|epom|imobile|supersonicads|loopme|applovin|pandora|mytarget|bidvertiser|chitika|popads|propellerads|buysellads|adhit|hilltopads|plugrush|popcash|popunder|revenuehits|trafficjunky|trafficfactory|zero-|smartoasis)(?:[-._][\w\d]*)*\.js$` commonJSLibraryFileRegexCompiled = regexp.MustCompile(CommonJSLibraryFileRegex) ) From 4d5c4b820a6591fbc50f0a4dd3e38e654d260937 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:29:48 +0000 Subject: [PATCH 069/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.0.57 to 0.1.1 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.57 to 0.1.1. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.57...v0.1.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 80336803..58aec6bf 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.57 + github.com/projectdiscovery/dsl v0.1.1 github.com/projectdiscovery/fastdialer v0.1.0 github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 diff --git a/go.sum b/go.sum index d21b019b..a53db506 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= -github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= +github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= +github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= From ad609826e68d502e2b8d66cd084272c8d8e81211 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:29:52 +0000 Subject: [PATCH 070/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.43 to 0.0.45 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.43 to 0.0.45. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.43...v0.0.45) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 80336803..65b89ff1 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/projectdiscovery/fastdialer v0.1.0 github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.43 + github.com/projectdiscovery/hmap v0.0.45 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.41 github.com/projectdiscovery/retryablehttp-go v1.0.61 diff --git a/go.sum b/go.sum index d21b019b..03d6faef 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,8 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg= -github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc= +github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U= +github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From 38b06eb74a9c85c910a0411ef0f100edb20608bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:55:18 +0000 Subject: [PATCH 071/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.41 to 0.0.44. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.41...v0.0.44) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 65b89ff1..e83f7b5e 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.45 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.41 + github.com/projectdiscovery/ratelimit v0.0.44 github.com/projectdiscovery/retryablehttp-go v1.0.61 github.com/projectdiscovery/useragent v0.0.53 github.com/projectdiscovery/utils v0.1.0 @@ -95,6 +95,7 @@ require ( github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/term v0.20.0 // indirect + golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index 03d6faef..c364acfe 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm2HFxguA8a+6E= -github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= +github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g= +github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE= github.com/projectdiscovery/retryabledns v1.0.61 h1:1WSX1wMZoI5IS6qdP1tgemLYtSIA7z98r1KnWY9PpbA= github.com/projectdiscovery/retryabledns v1.0.61/go.mod h1:XnGuCyuGaR3xCOsFCY/P6O9KMT/+rnN227OAkR8QQ+4= github.com/projectdiscovery/retryablehttp-go v1.0.61 h1:+LCmSWOAXd7p+S889TWhw1Fsjjdv58Ja3xL9gBUGWg8= @@ -465,6 +465,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= From 0deefbfe578f83c40b8c65d2ec8fc7906fea0ed6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:58:25 +0000 Subject: [PATCH 072/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.1.0...v0.1.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7c7d6504..e1bd2b34 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 - github.com/projectdiscovery/fastdialer v0.1.0 + github.com/projectdiscovery/fastdialer v0.1.1 github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.45 diff --git a/go.sum b/go.sum index 0644a232..1cdbea8b 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= -github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg= -github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w= +github.com/projectdiscovery/fastdialer v0.1.1 h1:51IovI02Ime7fVCWFxMpg1akCcaRHpVQnjpTQT0+0G0= +github.com/projectdiscovery/fastdialer v0.1.1/go.mod h1:tIMAT93FmV90EoVfM1lfaTrkNEbqT+G4jLyyoOO5McY= github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= From f2fceb6838464cabcc2afbc9e3ed58bb30990285 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:23:33 +0000 Subject: [PATCH 073/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.61 to 1.0.63. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.61...v1.0.63) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a974af70..1782da82 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.45 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.44 - github.com/projectdiscovery/retryablehttp-go v1.0.61 + github.com/projectdiscovery/retryablehttp-go v1.0.63 github.com/projectdiscovery/useragent v0.0.53 github.com/projectdiscovery/utils v0.1.0 github.com/projectdiscovery/wappalyzergo v0.1.2 diff --git a/go.sum b/go.sum index b5c7e133..d98a9333 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9Cdfr github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE= github.com/projectdiscovery/retryabledns v1.0.61 h1:1WSX1wMZoI5IS6qdP1tgemLYtSIA7z98r1KnWY9PpbA= github.com/projectdiscovery/retryabledns v1.0.61/go.mod h1:XnGuCyuGaR3xCOsFCY/P6O9KMT/+rnN227OAkR8QQ+4= -github.com/projectdiscovery/retryablehttp-go v1.0.61 h1:+LCmSWOAXd7p+S889TWhw1Fsjjdv58Ja3xL9gBUGWg8= -github.com/projectdiscovery/retryablehttp-go v1.0.61/go.mod h1:DGe6bUyvmhoMGifHA2SplUP8f8Bu87FCqZWADxj/YN8= +github.com/projectdiscovery/retryablehttp-go v1.0.63 h1:y/MpWrAO7702ybOLo/OuyAx6bZ6NJVEf85mGMAU51l8= +github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM= github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= From 73b9cc8a7d909eb4a8cfed29768939e96f983457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 5 Jun 2024 12:58:53 +0300 Subject: [PATCH 074/183] use request to construct a new one --- pkg/engine/hybrid/crawl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index c100d36d..63993ad2 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -68,7 +68,7 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re } else { statucCodeText = http.StatusText(statusCode) } - httpreq, err := http.NewRequest(e.Request.Method, URL.String(), strings.NewReader(e.Request.PostData)) + httpreq, err := http.NewRequest(request.Method, URL.String(), strings.NewReader(request.Body)) if err != nil { return errorutil.NewWithTag("hybrid", "could not new request").Wrap(err) } From ab8e87077811816151fcb4d039def27d1f7443ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 6 Jun 2024 15:35:55 +0300 Subject: [PATCH 075/183] Revert "use request to construct a new one" This reverts commit 73b9cc8a7d909eb4a8cfed29768939e96f983457. --- pkg/engine/hybrid/crawl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index 63993ad2..c100d36d 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -68,7 +68,7 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re } else { statucCodeText = http.StatusText(statusCode) } - httpreq, err := http.NewRequest(request.Method, URL.String(), strings.NewReader(request.Body)) + httpreq, err := http.NewRequest(e.Request.Method, URL.String(), strings.NewReader(e.Request.PostData)) if err != nil { return errorutil.NewWithTag("hybrid", "could not new request").Wrap(err) } From 2c59b601090d2c688a420f02cec2884b22b8eedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 6 Jun 2024 16:04:34 +0300 Subject: [PATCH 076/183] make standard request when it's form --- pkg/engine/hybrid/crawl.go | 94 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index c100d36d..45805180 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -2,6 +2,7 @@ package hybrid import ( "bytes" + "context" "io" "net/http" "net/http/httputil" @@ -26,6 +27,10 @@ import ( ) func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Request) (*navigation.Response, error) { + if request.Tag == "form" { + return c.makeStandardRequest(s, request) + } + depth := request.Depth + 1 response := &navigation.Response{ Depth: depth, @@ -233,6 +238,95 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re return response, nil } +// copy of the makeRequest method from standard/crawl.go +func (c *Crawler) makeStandardRequest(s *common.CrawlSession, request *navigation.Request) (*navigation.Response, error) { + response := &navigation.Response{ + Depth: request.Depth + 1, + RootHostname: s.Hostname, + } + ctx := context.WithValue(s.Ctx, navigation.Depth{}, request.Depth) + httpReq, err := http.NewRequestWithContext(ctx, request.Method, request.URL, nil) + if err != nil { + return response, err + } + if request.Body != "" && request.Method != "GET" { + httpReq.Body = io.NopCloser(strings.NewReader(request.Body)) + } + req, err := retryablehttp.FromRequest(httpReq) + if err != nil { + return response, err + } + req.Header.Set("User-Agent", utils.WebUserAgent()) + + // Set the headers for the request. + for k, v := range request.Headers { + req.Header.Set(k, v) + if k == "Host" { + req.Host = v + } + } + for k, v := range c.Headers { + req.Header.Set(k, v) + if k == "Host" { + req.Host = v + } + } + + resp, err := s.HttpClient.Do(req) + if resp != nil { + defer func() { + if resp.Body != nil && resp.StatusCode != http.StatusSwitchingProtocols { + _, _ = io.Copy(io.Discard, resp.Body) + } + _ = resp.Body.Close() + }() + } + + rawRequestBytes, _ := req.Dump() + request.Raw = string(rawRequestBytes) + + if err != nil { + return response, err + } + if resp.StatusCode == http.StatusSwitchingProtocols { + return response, nil + } + limitReader := io.LimitReader(resp.Body, int64(c.Options.Options.BodyReadSize)) + data, err := io.ReadAll(limitReader) + if err != nil { + return response, err + } + if !c.Options.UniqueFilter.UniqueContent(data) { + return &navigation.Response{}, nil + } + + technologies := c.Options.Wappalyzer.Fingerprint(resp.Header, data) + response.Technologies = mapsutil.GetKeys(technologies) + + resp.Body = io.NopCloser(strings.NewReader(string(data))) + + response.Body = string(data) + response.Resp = resp + response.Reader, err = goquery.NewDocumentFromReader(bytes.NewReader(data)) + response.Reader.Url, _ = url.Parse(request.URL) + response.StatusCode = resp.StatusCode + response.Headers = utils.FlattenHeaders(resp.Header) + if c.Options.Options.FormExtraction { + response.Forms = append(response.Forms, utils.ParseFormFields(response.Reader)...) + } + + resp.ContentLength = int64(len(data)) + + rawResponseBytes, _ := httputil.DumpResponse(resp, true) + response.Raw = string(rawResponseBytes) + + if err != nil { + return response, errorutil.NewWithTag("hybrid", "could not make document from reader").Wrap(err) + } + + return response, nil +} + func (c *Crawler) addHeadersToPage(page *rod.Page) { if len(c.Headers) == 0 { return From 13e9072c50d7897b18ee6f74d74467970f58986b Mon Sep 17 00:00:00 2001 From: "alban.stourbe stourbe" Date: Thu, 6 Jun 2024 17:30:57 +0200 Subject: [PATCH 077/183] Add AutoFillSuggestion for select,textarea in form and perfom autofill test --- pkg/engine/parser/parser.go | 10 +-- pkg/utils/formfill.go | 169 ++++++++++++++++++++++++++++++++++++ pkg/utils/formfill_test.go | 32 +++++-- 3 files changed, 200 insertions(+), 11 deletions(-) diff --git a/pkg/engine/parser/parser.go b/pkg/engine/parser/parser.go index d621a6de..9fe442ab 100644 --- a/pkg/engine/parser/parser.go +++ b/pkg/engine/parser/parser.go @@ -527,16 +527,16 @@ func bodyFormTagParser(resp *navigation.Response) (navigationRequests []*navigat multipartWriter = multipart.NewWriter(&sb) } - // Get the form field suggestions for all inputs - formInputs := []utils.FormInput{} - item.Find("input").Each(func(index int, item *goquery.Selection) { + // Get the form field suggestions for all elements in the form + formFields := []interface{}{} + item.Find("input, select, textarea").Each(func(index int, item *goquery.Selection) { if len(item.Nodes) == 0 { return } - formInputs = append(formInputs, utils.ConvertGoquerySelectionToFormInput(item)) + formFields = append(formFields, utils.ConvertGoquerySelectionToFormField(item)) }) - dataMap := utils.FormInputFillSuggestions(formInputs) + dataMap := utils.FormFillSuggestions(formFields) dataMap.Iterate(func(key, value string) bool { if key == "" { return true diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 1ace9a35..57f4e56d 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -41,6 +41,25 @@ type FormInput struct { Attributes mapsutil.OrderedMap[string, string] } +// FormOption is an option for a select input +type FormOption struct { + Value string + Selected string + Attributes mapsutil.OrderedMap[string, string] +} + +// FormSelect is a select input for a form field +type FormSelect struct { + Name string + Attributes mapsutil.OrderedMap[string, string] + FormOptions []FormOption +} + +type FormTextArea struct { + Name string + Attributes mapsutil.OrderedMap[string, string] +} + // FormInputFillSuggestions returns a list of form filling suggestions // for inputs returning the specified recommended values. func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, string] { @@ -106,6 +125,78 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st return data } +// FormSelectFill fills a map with selected values from a slice of FormSelect structs. +// It iterates over each FormSelect struct in the inputs slice and checks for a selected option. +// If a selected option is found, it adds the corresponding value to the map using the input's name as the key. +// If no option is selected, it selects the first option and adds its value to the map. +// The function returns the filled map. +func FormSelectFill(inputs []FormSelect) mapsutil.OrderedMap[string, string] { + data := mapsutil.NewOrderedMap[string, string]() + for _, input := range inputs { + for _, option := range input.FormOptions { + if option.Selected != "" { + data.Set(input.Name, option.Value) + break + } + } + + // If no option is selected, select the first one + if !data.Has(input.Name) && len(input.FormOptions) > 0 { + data.Set(input.Name, input.FormOptions[0].Value) + } + } + return data +} + +// FormTextAreaFill fills the form text areas with placeholder values. +// It takes a slice of FormTextArea structs as input and returns an OrderedMap +// containing the form field names as keys and the placeholder values as values. +func FormTextAreaFill(inputs []FormTextArea) mapsutil.OrderedMap[string, string] { + data := mapsutil.NewOrderedMap[string, string]() + for _, input := range inputs { + data.Set(input.Name, FormData.Placeholder) + } + return data +} + +// FormFillSuggestions takes a slice of form fields and returns an ordered map +// containing suggestions for filling those form fields. The function iterates +// over each form field and based on its type, calls the corresponding fill +// function to generate suggestions. The suggestions are then merged into a +// single ordered map and returned. +// +// Parameters: +// - formFields: A slice of form fields. +// +// Returns: +// An ordered map containing suggestions for filling the form fields. +func FormFillSuggestions(formFields []interface{}) mapsutil.OrderedMap[string, string] { + merged := mapsutil.NewOrderedMap[string, string]() + for _, item := range formFields { + switch v := item.(type) { + case FormInput: + dataMapInputs := FormInputFillSuggestions([]FormInput{v}) + dataMapInputs.Iterate(func(key, value string) bool { + merged.Set(key, value) + return true + }) + case FormSelect: + dataMapSelects := FormSelectFill([]FormSelect{v}) + dataMapSelects.Iterate(func(key, value string) bool { + merged.Set(key, value) + return true + }) + case FormTextArea: + dataMapTextArea := FormTextAreaFill([]FormTextArea{v}) + dataMapTextArea.Iterate(func(key, value string) bool { + merged.Set(key, value) + return true + }) + } + } + return merged +} + // ConvertGoquerySelectionToFormInput converts goquery selection to form input func ConvertGoquerySelectionToFormInput(item *goquery.Selection) FormInput { attrs := item.Nodes[0].Attr @@ -125,3 +216,81 @@ func ConvertGoquerySelectionToFormInput(item *goquery.Selection) FormInput { } return input } + +// ConvertGoquerySelectionToFormOption converts a goquery.Selection object to a FormOption object. +// It extracts the attributes from the goquery.Selection object and populates a FormOption object with the extracted values. +func ConvertGoquerySelectionToFormOption(item *goquery.Selection) FormOption { + attrs := item.Nodes[0].Attr + input := FormOption{Attributes: mapsutil.NewOrderedMap[string, string]()} + for _, attribute := range attrs { + switch attribute.Key { + case "value": + input.Value = attribute.Val + + case "selected": + input.Selected = attribute.Key + default: + input.Attributes.Set(attribute.Key, attribute.Val) + } + } + return input +} + +// ConvertGoquerySelectionToFormSelect converts a goquery.Selection object to a FormSelect object. +// It extracts the attributes and form options from the goquery.Selection and populates them in the FormSelect object. +// The converted FormSelect object is then returned. +func ConvertGoquerySelectionToFormSelect(item *goquery.Selection) FormSelect { + attrs := item.Nodes[0].Attr + input := FormSelect{Attributes: mapsutil.NewOrderedMap[string, string]()} + for _, attribute := range attrs { + switch attribute.Key { + case "name": + input.Name = attribute.Val + default: + input.Attributes.Set(attribute.Key, attribute.Val) + } + } + + input.FormOptions = []FormOption{} + item.Find("option").Each(func(_ int, option *goquery.Selection) { + input.FormOptions = append(input.FormOptions, ConvertGoquerySelectionToFormOption(option)) + }) + return input +} + +// ConvertGoquerySelectionToFormTextArea converts a goquery.Selection object to a FormTextArea struct. +// It extracts the attributes from the first node of the selection and populates a FormTextArea object with the extracted data. +// The "name" attribute is assigned to the Name field of the FormTextArea, while other attributes are added to the Attributes map. +func ConvertGoquerySelectionToFormTextArea(item *goquery.Selection) FormTextArea { + attrs := item.Nodes[0].Attr + input := FormTextArea{Attributes: mapsutil.NewOrderedMap[string, string]()} + for _, attribute := range attrs { + switch attribute.Key { + case "name": + input.Name = attribute.Val + default: + input.Attributes.Set(attribute.Key, attribute.Val) + } + } + return input +} + +// ConvertGoquerySelectionToFormField converts a goquery.Selection object to a form field. +// It checks the type of the selection and calls the appropriate conversion function. +// If the selection is an input, it calls ConvertGoquerySelectionToFormInput. +// If the selection is a select, it calls ConvertGoquerySelectionToFormSelect. +// If the selection is a textarea, it calls ConvertGoquerySelectionToFormTextArea. +// If the selection is of any other type, it returns nil. +func ConvertGoquerySelectionToFormField(item *goquery.Selection) interface{} { + if item.Is("input") { + return ConvertGoquerySelectionToFormInput(item) + } + if item.Is("select") { + return ConvertGoquerySelectionToFormSelect(item) + } + if item.Is("textarea") { + return ConvertGoquerySelectionToFormTextArea(item) + } + + return nil +} diff --git a/pkg/utils/formfill_test.go b/pkg/utils/formfill_test.go index 268f654c..10de174b 100644 --- a/pkg/utils/formfill_test.go +++ b/pkg/utils/formfill_test.go @@ -33,7 +33,27 @@ var htmlFormInputExample = ` -

+

Kindly Select your favourite food

+ +

Kindly Select your favourite country

+ + + + + +

+ ` @@ -44,16 +64,16 @@ func TestFormInputFillSuggestions(t *testing.T) { document.Find("form[action]").Each(func(i int, item *goquery.Selection) { queryValuesWriter := make(url.Values) - formInputs := []FormInput{} + formFields := []interface{}{} - item.Find("input").Each(func(index int, item *goquery.Selection) { + item.Find("input, textarea, select").Each(func(index int, item *goquery.Selection) { if len(item.Nodes) == 0 { return } - formInputs = append(formInputs, ConvertGoquerySelectionToFormInput(item)) + formFields = append(formFields, ConvertGoquerySelectionToFormField(item)) }) - dataMap := FormInputFillSuggestions(formInputs) + dataMap := FormFillSuggestions(formFields) dataMap.Iterate(func(key, value string) bool { if key == "" || value == "" { return true @@ -62,6 +82,6 @@ func TestFormInputFillSuggestions(t *testing.T) { return true }) value := queryValuesWriter.Encode() - require.Equal(t, "Startdate=katana&color=red&firstname=katana&num=51&password=katana&sport1=cricket&sport2=tennis&sport3=football&telephone=katanaP%40assw0rd1&upclick=%23a52a2a", value, "could not get correct encoded form") + require.Equal(t, "Startdate=katana&color=green&country=india&firstname=katana&food=pasta&message=katana&num=51&password=katana&sport1=cricket&sport2=tennis&sport3=football&telephone=katanaP%40assw0rd1&upclick=%23a52a2a", value, "could not get correct encoded form") }) } From b366f3ffc9aa1e0bc556a3a099eab5d1e923a162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 7 Jun 2024 06:55:59 +0300 Subject: [PATCH 078/183] Revert "make standard request when it's form" This reverts commit 2c59b601090d2c688a420f02cec2884b22b8eedb. --- pkg/engine/hybrid/crawl.go | 94 -------------------------------------- 1 file changed, 94 deletions(-) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index 45805180..c100d36d 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -2,7 +2,6 @@ package hybrid import ( "bytes" - "context" "io" "net/http" "net/http/httputil" @@ -27,10 +26,6 @@ import ( ) func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Request) (*navigation.Response, error) { - if request.Tag == "form" { - return c.makeStandardRequest(s, request) - } - depth := request.Depth + 1 response := &navigation.Response{ Depth: depth, @@ -238,95 +233,6 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re return response, nil } -// copy of the makeRequest method from standard/crawl.go -func (c *Crawler) makeStandardRequest(s *common.CrawlSession, request *navigation.Request) (*navigation.Response, error) { - response := &navigation.Response{ - Depth: request.Depth + 1, - RootHostname: s.Hostname, - } - ctx := context.WithValue(s.Ctx, navigation.Depth{}, request.Depth) - httpReq, err := http.NewRequestWithContext(ctx, request.Method, request.URL, nil) - if err != nil { - return response, err - } - if request.Body != "" && request.Method != "GET" { - httpReq.Body = io.NopCloser(strings.NewReader(request.Body)) - } - req, err := retryablehttp.FromRequest(httpReq) - if err != nil { - return response, err - } - req.Header.Set("User-Agent", utils.WebUserAgent()) - - // Set the headers for the request. - for k, v := range request.Headers { - req.Header.Set(k, v) - if k == "Host" { - req.Host = v - } - } - for k, v := range c.Headers { - req.Header.Set(k, v) - if k == "Host" { - req.Host = v - } - } - - resp, err := s.HttpClient.Do(req) - if resp != nil { - defer func() { - if resp.Body != nil && resp.StatusCode != http.StatusSwitchingProtocols { - _, _ = io.Copy(io.Discard, resp.Body) - } - _ = resp.Body.Close() - }() - } - - rawRequestBytes, _ := req.Dump() - request.Raw = string(rawRequestBytes) - - if err != nil { - return response, err - } - if resp.StatusCode == http.StatusSwitchingProtocols { - return response, nil - } - limitReader := io.LimitReader(resp.Body, int64(c.Options.Options.BodyReadSize)) - data, err := io.ReadAll(limitReader) - if err != nil { - return response, err - } - if !c.Options.UniqueFilter.UniqueContent(data) { - return &navigation.Response{}, nil - } - - technologies := c.Options.Wappalyzer.Fingerprint(resp.Header, data) - response.Technologies = mapsutil.GetKeys(technologies) - - resp.Body = io.NopCloser(strings.NewReader(string(data))) - - response.Body = string(data) - response.Resp = resp - response.Reader, err = goquery.NewDocumentFromReader(bytes.NewReader(data)) - response.Reader.Url, _ = url.Parse(request.URL) - response.StatusCode = resp.StatusCode - response.Headers = utils.FlattenHeaders(resp.Header) - if c.Options.Options.FormExtraction { - response.Forms = append(response.Forms, utils.ParseFormFields(response.Reader)...) - } - - resp.ContentLength = int64(len(data)) - - rawResponseBytes, _ := httputil.DumpResponse(resp, true) - response.Raw = string(rawResponseBytes) - - if err != nil { - return response, errorutil.NewWithTag("hybrid", "could not make document from reader").Wrap(err) - } - - return response, nil -} - func (c *Crawler) addHeadersToPage(page *rod.Page) { if len(c.Headers) == 0 { return From 3618c92e71450dd63fbdb8bc4b466c0a5e6a941f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 7 Jun 2024 07:01:30 +0300 Subject: [PATCH 079/183] disable -aff for -hl --- internal/runner/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/runner/options.go b/internal/runner/options.go index 0c7fd0fb..eb7f1738 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,6 +25,13 @@ func validateOptions(options *types.Options) error { return errorutil.New("no inputs specified for crawler") } + // Disabling automatic form fill (-aff) for headless navigation due to incorrect implementation. + // Form filling should be handled via headless actions within the page context + if options.Headless && options.AutomaticFormFill { + options.AutomaticFormFill = false + gologger.Info().Msgf("Automatic form fill (-aff) has been disabled for headless navigation.") + } + if options.Headless && options.Passive { return errorutil.New("headless mode (-headless) and passive mode (-passive) cannot be used together") } From c60b49a7f7ca2f23c122dcbfb64b2d09845c1695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 7 Jun 2024 13:49:17 +0300 Subject: [PATCH 080/183] fix header marshalling --- pkg/navigation/response.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/navigation/response.go b/pkg/navigation/response.go index a9dd2b7e..2ecbd5a5 100644 --- a/pkg/navigation/response.go +++ b/pkg/navigation/response.go @@ -20,7 +20,7 @@ type Form struct { func (h *Headers) MarshalJSON() ([]byte, error) { hCopy := make(Headers) for k, v := range *h { - k := strings.ReplaceAll(strings.ToLower(k), "-", "_") + k := strings.ToLower(k) hCopy[k] = v } return jsoniter.Marshal(hCopy) From 45ac622ccec7325eba19574e357718a48988e556 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:40:24 +0000 Subject: [PATCH 081/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.45 to 0.0.46 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.45 to 0.0.46. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.45...v0.0.46) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b15a707e..7cb6edc9 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/projectdiscovery/fastdialer v0.1.1 github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.45 + github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.44 github.com/projectdiscovery/retryablehttp-go v1.0.63 github.com/projectdiscovery/useragent v0.0.53 - github.com/projectdiscovery/utils v0.1.0 + github.com/projectdiscovery/utils v0.1.1 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index b3e88602..9bc7aa5a 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,8 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.45 h1:D3PinywmpDtIX2lc9bnqaRJoFDPCCxUf4RvHA3Gdr6U= -github.com/projectdiscovery/hmap v0.0.45/go.mod h1:vJ7HYzhDnOjsEc9d43Q2pIWRJ5QvFXdObTTQvaUZaPk= +github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk= +github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.63 h1:y/MpWrAO7702ybOLo/OuyAx6 github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM= github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= -github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= -github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= +github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4= +github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= github.com/projectdiscovery/wappalyzergo v0.1.2/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 4ebcb40fafee1ef393e83fd3f484fd61108e43ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:40:38 +0000 Subject: [PATCH 082/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.63 to 1.0.64. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.63...v1.0.64) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b15a707e..6a14da5e 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/projectdiscovery/hmap v0.0.45 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.44 - github.com/projectdiscovery/retryablehttp-go v1.0.63 + github.com/projectdiscovery/retryablehttp-go v1.0.64 github.com/projectdiscovery/useragent v0.0.53 - github.com/projectdiscovery/utils v0.1.0 + github.com/projectdiscovery/utils v0.1.1 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -122,7 +122,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.61 // indirect + github.com/projectdiscovery/retryabledns v1.0.62 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index b3e88602..33ebcabe 100644 --- a/go.sum +++ b/go.sum @@ -226,14 +226,14 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g= github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE= -github.com/projectdiscovery/retryabledns v1.0.61 h1:1WSX1wMZoI5IS6qdP1tgemLYtSIA7z98r1KnWY9PpbA= -github.com/projectdiscovery/retryabledns v1.0.61/go.mod h1:XnGuCyuGaR3xCOsFCY/P6O9KMT/+rnN227OAkR8QQ+4= -github.com/projectdiscovery/retryablehttp-go v1.0.63 h1:y/MpWrAO7702ybOLo/OuyAx6bZ6NJVEf85mGMAU51l8= -github.com/projectdiscovery/retryablehttp-go v1.0.63/go.mod h1:ytE1KGi63NFAyLzWqTwZJXwjAg7apgqTnJtaZeZ4qtM= +github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA= +github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= +github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= +github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= -github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps= -github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40= +github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4= +github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= github.com/projectdiscovery/wappalyzergo v0.1.2/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 40fa8a13979d06b15e19006c5b02a84295116f57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:09:31 +0000 Subject: [PATCH 083/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.44 to 0.0.45. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.44...v0.0.45) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 07b1aed4..ee53d9ae 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.44 + github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 github.com/projectdiscovery/useragent v0.0.53 github.com/projectdiscovery/utils v0.1.1 diff --git a/go.sum b/go.sum index c460055e..07596510 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g= -github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE= +github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= +github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA= github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= From 6e51f6c6e1a8a4b8628ad8b91567005bcde3e673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:36:51 +0000 Subject: [PATCH 084/183] chore(deps): bump github.com/projectdiscovery/utils from 0.1.0 to 0.1.3 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.1.0 to 0.1.3. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.1.0...v0.1.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ee53d9ae..b7fafe38 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 - github.com/projectdiscovery/fastdialer v0.1.1 + github.com/projectdiscovery/fastdialer v0.1.2 github.com/projectdiscovery/goflags v0.1.53 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.46 @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 github.com/projectdiscovery/useragent v0.0.53 - github.com/projectdiscovery/utils v0.1.1 + github.com/projectdiscovery/utils v0.1.3 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 07596510..58c81810 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= -github.com/projectdiscovery/fastdialer v0.1.1 h1:51IovI02Ime7fVCWFxMpg1akCcaRHpVQnjpTQT0+0G0= -github.com/projectdiscovery/fastdialer v0.1.1/go.mod h1:tIMAT93FmV90EoVfM1lfaTrkNEbqT+G4jLyyoOO5McY= +github.com/projectdiscovery/fastdialer v0.1.2 h1:wNUxeMIwiagXvseCmhw5qlnACJV6kOAPtAGswjqNyWE= +github.com/projectdiscovery/fastdialer v0.1.2/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qse github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= -github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4= -github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o= +github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg= +github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= github.com/projectdiscovery/wappalyzergo v0.1.2/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From fd098d5772da8bf0a890d94f203df9236b118691 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:06:44 +0000 Subject: [PATCH 085/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.53 to 0.0.55. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.53...v0.0.55) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b7fafe38..13182361 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 github.com/projectdiscovery/fastdialer v0.1.2 - github.com/projectdiscovery/goflags v0.1.53 + github.com/projectdiscovery/goflags v0.1.54 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 - github.com/projectdiscovery/useragent v0.0.53 + github.com/projectdiscovery/useragent v0.0.55 github.com/projectdiscovery/utils v0.1.3 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 @@ -50,6 +50,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hdm/jarm-go v0.0.7 // indirect diff --git a/go.sum b/go.sum index 58c81810..2e72d84c 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -210,8 +212,8 @@ github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJ github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= github.com/projectdiscovery/fastdialer v0.1.2 h1:wNUxeMIwiagXvseCmhw5qlnACJV6kOAPtAGswjqNyWE= github.com/projectdiscovery/fastdialer v0.1.2/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= -github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE= -github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380= +github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8= +github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -230,8 +232,8 @@ github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8ed github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= -github.com/projectdiscovery/useragent v0.0.53 h1:BKe6z9vb+ajNqmItupjuMO0B6ulG1xC26RsVgEE5p/k= -github.com/projectdiscovery/useragent v0.0.53/go.mod h1:rToNGuRWpwXACjdL34DuuI0LLHPrCwfWwYHeWMvtPkw= +github.com/projectdiscovery/useragent v0.0.55 h1:SBeH+O9CTAgqRN96bAaOVw/ZyIILnMmcxE9xc+zWRYY= +github.com/projectdiscovery/useragent v0.0.55/go.mod h1:WCUCKjalMFtnn7Xq2z33/CoWm1gbO9jeZkFLEFC6ChA= github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg= github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= From e63a02604917e46bfcf1774ffc2a0f497ff0134b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:02:51 +0000 Subject: [PATCH 086/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.55 to 0.0.56. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.55...v0.0.56) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 13182361..c331efb8 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 github.com/projectdiscovery/fastdialer v0.1.2 - github.com/projectdiscovery/goflags v0.1.54 + github.com/projectdiscovery/goflags v0.1.55 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 - github.com/projectdiscovery/useragent v0.0.55 + github.com/projectdiscovery/useragent v0.0.56 github.com/projectdiscovery/utils v0.1.3 github.com/projectdiscovery/wappalyzergo v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 2e72d84c..7627e0e1 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJ github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= github.com/projectdiscovery/fastdialer v0.1.2 h1:wNUxeMIwiagXvseCmhw5qlnACJV6kOAPtAGswjqNyWE= github.com/projectdiscovery/fastdialer v0.1.2/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= -github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8= -github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc= +github.com/projectdiscovery/goflags v0.1.55 h1:oVx7ZhNd72g8P3MjRZTb+JmkCf6F40il75b2LrZDfWo= +github.com/projectdiscovery/goflags v0.1.55/go.mod h1:0VWRorAiIBbOuKpbFVbwiSUfzrRsyaEI0z2nOSRT/Zg= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -232,8 +232,8 @@ github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8ed github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= -github.com/projectdiscovery/useragent v0.0.55 h1:SBeH+O9CTAgqRN96bAaOVw/ZyIILnMmcxE9xc+zWRYY= -github.com/projectdiscovery/useragent v0.0.55/go.mod h1:WCUCKjalMFtnn7Xq2z33/CoWm1gbO9jeZkFLEFC6ChA= +github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ= +github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s= github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg= github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= From d04f8a0ec7cd546507a21a88a1947f84ac04507e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:30:49 +0000 Subject: [PATCH 087/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.2 to 0.1.6. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.2...v0.1.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c331efb8..3cbad228 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.64 github.com/projectdiscovery/useragent v0.0.56 github.com/projectdiscovery/utils v0.1.3 - github.com/projectdiscovery/wappalyzergo v0.1.2 + github.com/projectdiscovery/wappalyzergo v0.1.6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 7627e0e1..c33b80de 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2R github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s= github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg= github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= -github.com/projectdiscovery/wappalyzergo v0.1.2 h1:GoTGetBiKQEoN4lfxJFuxQrBkyby1uHYHePDhY4jNQc= -github.com/projectdiscovery/wappalyzergo v0.1.2/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= +github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M= +github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From 8243e7d9370c0e6eb84f92249b57c9846aca62ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:31:06 +0000 Subject: [PATCH 088/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.62 to 1.0.64. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.62...v1.0.64) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c331efb8..8db64553 100644 --- a/go.mod +++ b/go.mod @@ -123,7 +123,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.62 // indirect + github.com/projectdiscovery/retryabledns v1.0.64 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 7627e0e1..17714e0f 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= -github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA= -github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= +github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= +github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ= From 97e3f03494489bd46cba2ec2e165e91161982f8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:31:08 +0000 Subject: [PATCH 089/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.54 to 0.1.56. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.54...v0.1.56) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c331efb8..8baa7268 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 github.com/projectdiscovery/fastdialer v0.1.2 - github.com/projectdiscovery/goflags v0.1.55 + github.com/projectdiscovery/goflags v0.1.56 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.46 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index 7627e0e1..afd1d019 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJ github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= github.com/projectdiscovery/fastdialer v0.1.2 h1:wNUxeMIwiagXvseCmhw5qlnACJV6kOAPtAGswjqNyWE= github.com/projectdiscovery/fastdialer v0.1.2/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= -github.com/projectdiscovery/goflags v0.1.55 h1:oVx7ZhNd72g8P3MjRZTb+JmkCf6F40il75b2LrZDfWo= -github.com/projectdiscovery/goflags v0.1.55/go.mod h1:0VWRorAiIBbOuKpbFVbwiSUfzrRsyaEI0z2nOSRT/Zg= +github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= +github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From 9c20851c62e82ffb5c0cd2582218d605652e776d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:59:14 +0000 Subject: [PATCH 090/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.46 to 0.0.47 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.46 to 0.0.47. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.46...v0.0.47) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5bd43f92..725480b7 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/projectdiscovery/fastdialer v0.1.2 github.com/projectdiscovery/goflags v0.1.56 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.46 + github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 diff --git a/go.sum b/go.sum index d7009fa3..97dd80a3 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,8 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk= -github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8= +github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk= +github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From 8cb4d10adcb20349f5b29fea4c446a8227fb9082 Mon Sep 17 00:00:00 2001 From: "alban.stourbe stourbe" Date: Tue, 18 Jun 2024 11:41:54 +0200 Subject: [PATCH 091/183] New way to add headers for hybrid request output --- pkg/engine/hybrid/crawl.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index f3a7d628..d2bfad81 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -74,9 +74,11 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re } // Note: headers are originally sent using `c.addHeadersToPage` below changes are done so that // headers are reflected in request dump + // Headers, CustomHeaders, and Cookies are present in e.Request.Headers. We need to consider all of them and not only CustomHeaders + // Otherwise, we will miss headers and output will be inconsistent if httpreq != nil { - for k, v := range c.Headers { - httpreq.Header.Set(k, v) + for k, v := range e.Request.Headers { + httpreq.Header.Set(k, v.String()) } } From 6d2c793f6a8aaea529f851285a05d474f738aee8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:12:08 +0000 Subject: [PATCH 092/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.56 to 0.0.57. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.56...v0.0.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 725480b7..3db8a47c 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 - github.com/projectdiscovery/fastdialer v0.1.2 + github.com/projectdiscovery/fastdialer v0.1.3 github.com/projectdiscovery/goflags v0.1.56 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.64 - github.com/projectdiscovery/useragent v0.0.56 - github.com/projectdiscovery/utils v0.1.3 + github.com/projectdiscovery/useragent v0.0.57 + github.com/projectdiscovery/utils v0.1.4 github.com/projectdiscovery/wappalyzergo v0.1.6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 97dd80a3..79be1085 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= -github.com/projectdiscovery/fastdialer v0.1.2 h1:wNUxeMIwiagXvseCmhw5qlnACJV6kOAPtAGswjqNyWE= -github.com/projectdiscovery/fastdialer v0.1.2/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= +github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI= +github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -232,10 +232,10 @@ github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfu github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= -github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ= -github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s= -github.com/projectdiscovery/utils v0.1.3 h1:yhHkrbYZA1eOO8e+fPDUvRMS5aUIalyM3Nab7rK4tpg= -github.com/projectdiscovery/utils v0.1.3/go.mod h1:gny8RbNYXE55IoamF6thRDQ8tcJEw+r0FOGAvncz/oQ= +github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= +github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= +github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI= +github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM= github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M= github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From de26006edcf364d37bc6f33ede2178560040b49d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:38:25 +0000 Subject: [PATCH 093/183] chore(deps): bump github.com/projectdiscovery/utils from 0.1.3 to 0.1.5 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.1.3 to 0.1.5. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.1.3...v0.1.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 11 ++++++----- go.sum | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 3db8a47c..fc770828 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 - github.com/projectdiscovery/fastdialer v0.1.3 + github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.56 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 - github.com/projectdiscovery/retryablehttp-go v1.0.64 + github.com/projectdiscovery/retryablehttp-go v1.0.65 github.com/projectdiscovery/useragent v0.0.57 - github.com/projectdiscovery/utils v0.1.4 + github.com/projectdiscovery/utils v0.1.5 github.com/projectdiscovery/wappalyzergo v0.1.6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -39,6 +39,7 @@ require ( github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -46,6 +47,7 @@ require ( github.com/dlclark/regexp2 v1.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/gaissmai/bart v0.9.5 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -122,7 +124,7 @@ require ( github.com/nwaples/rardecode v1.1.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/networkpolicy v0.0.8 + github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/retryabledns v1.0.64 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -131,7 +133,6 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect diff --git a/go.sum b/go.sum index 79be1085..d1325924 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= -github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -66,6 +66,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= +github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -210,8 +212,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= -github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI= -github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= +github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= +github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -224,18 +226,18 @@ github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScL github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= -github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= -github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= +github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= +github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= -github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= -github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= +github.com/projectdiscovery/retryablehttp-go v1.0.65 h1:GCZI9CUbYkldy9Iup4C7w6aVTxZuAHpSZTVy92qHPo4= +github.com/projectdiscovery/retryablehttp-go v1.0.65/go.mod h1:qF0UuglJiYJcezEoMIyYVYhHGwaOBy8bh3gRGz7CBs8= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= -github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI= -github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM= +github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= +github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M= github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 9f1044870e359e7b29e1d2703700dd596080aabd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:38:26 +0000 Subject: [PATCH 094/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.1.2 to 0.1.5. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.1.2...v0.1.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 7 ++++--- go.sum | 14 ++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 3db8a47c..a95bed7c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 - github.com/projectdiscovery/fastdialer v0.1.3 + github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.56 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.47 @@ -39,6 +39,7 @@ require ( github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -46,6 +47,7 @@ require ( github.com/dlclark/regexp2 v1.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/gaissmai/bart v0.9.5 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -122,7 +124,7 @@ require ( github.com/nwaples/rardecode v1.1.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/networkpolicy v0.0.8 + github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/retryabledns v1.0.64 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -131,7 +133,6 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect diff --git a/go.sum b/go.sum index 79be1085..abf8e8e5 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= -github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -66,6 +66,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= +github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -210,8 +212,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= -github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI= -github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= +github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= +github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= @@ -224,8 +226,8 @@ github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScL github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= -github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= -github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= +github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= +github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= From 68c9035358e50421ba890cf5c85d2914eb8878f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:06:25 +0000 Subject: [PATCH 095/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.56 to 0.1.57. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.56...v0.1.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fc770828..ababb1d7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.1 github.com/projectdiscovery/fastdialer v0.1.5 - github.com/projectdiscovery/goflags v0.1.56 + github.com/projectdiscovery/goflags v0.1.57 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index d1325924..c51c996b 100644 --- a/go.sum +++ b/go.sum @@ -214,8 +214,8 @@ github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJ github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= -github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= -github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= +github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= +github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From 0776a790f78be9743450274d71e0d6ad23b8b432 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 25 Jun 2024 22:43:05 +0530 Subject: [PATCH 096/183] misc --- go.mod | 17 +++++++++-------- go.sum | 39 ++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index ababb1d7..950ecba5 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.65 github.com/projectdiscovery/useragent v0.0.57 github.com/projectdiscovery/utils v0.1.5 - github.com/projectdiscovery/wappalyzergo v0.1.6 + github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -97,7 +97,8 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.20.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/term v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect @@ -140,12 +141,12 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index c51c996b..2b22615f 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= @@ -238,8 +239,8 @@ github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpu github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= -github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M= -github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= +github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= +github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -380,15 +381,15 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -407,8 +408,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -418,8 +419,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -448,16 +449,16 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -467,8 +468,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -476,8 +477,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d835f94344ffe617d2048fb811b40dd602730f26 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 27 Jun 2024 10:51:37 +0300 Subject: [PATCH 097/183] fix cli log config issue (#830) --- internal/runner/runner.go | 2 +- pkg/types/crawler_options.go | 20 +------------------- pkg/types/options.go | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 03d9118e..1b91133e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -56,7 +56,7 @@ func New(options *types.Options) (*Runner, error) { } options.URLs = mapsutil.GetKeys(runnerState.InFlightUrls.GetAll()) } - + options.ConfigureOutput() showBanner() if options.Version { diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index ff490f4a..05dc4c9d 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -6,15 +6,12 @@ import ( "time" "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/output" "github.com/projectdiscovery/katana/pkg/utils/extensions" "github.com/projectdiscovery/katana/pkg/utils/filters" "github.com/projectdiscovery/katana/pkg/utils/scope" "github.com/projectdiscovery/ratelimit" errorutil "github.com/projectdiscovery/utils/errors" - logutil "github.com/projectdiscovery/utils/log" urlutil "github.com/projectdiscovery/utils/url" wappalyzer "github.com/projectdiscovery/wappalyzergo" ) @@ -42,7 +39,7 @@ type CrawlerOptions struct { // NewCrawlerOptions creates a new crawler options structure // from user specified options. func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { - configureOutput(options) + options.ConfigureOutput() extensionsValidator := extensions.NewValidator(options.ExtensionsMatch, options.ExtensionFilter) dialerOpts := fastdialer.DefaultOptions @@ -152,18 +149,3 @@ func (c *CrawlerOptions) ValidateScope(absURL, rootHostname string) (bool, error } return true, nil } - -// configureOutput configures the output logging levels to be displayed on the screen -func configureOutput(options *Options) { - if options.Silent { - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - } else if options.Verbose { - gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) - } else if options.Debug { - gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - } else { - gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) - } - - logutil.DisableDefaultLogger() -} diff --git a/pkg/types/options.go b/pkg/types/options.go index 5743be82..b402d628 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -6,8 +6,11 @@ import ( "time" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/output" fileutil "github.com/projectdiscovery/utils/file" + logutil "github.com/projectdiscovery/utils/log" ) // OnResultCallback (output.Result) @@ -194,3 +197,18 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { func (options *Options) ShouldResume() bool { return options.Resume != "" && fileutil.FileExists(options.Resume) } + +// ConfigureOutput configures the output logging levels to be displayed on the screen +func (options *Options) ConfigureOutput() { + if options.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } else if options.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) + } else if options.Debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } else { + gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) + } + + logutil.DisableDefaultLogger() +} From 5ae80aec1c2650054c0b3f27f671e3d40a5bbbdc Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:16:40 +0300 Subject: [PATCH 098/183] add `-store-field-dir` flag (#877) --- README.md | 3 ++- cmd/katana/main.go | 1 + pkg/output/fields.go | 3 +-- pkg/output/options.go | 1 + pkg/output/output.go | 12 ++++++++---- pkg/types/crawler_options.go | 1 + pkg/types/options.go | 2 ++ 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2011262e..e24af9b2 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ OUTPUT: -o, -output string file to write output to -sr, -store-response store http requests/responses -srd, -store-response-dir string store http requests/responses to custom directory + -sfd, -store-field-dir string store per-host field to custom directory -or, -omit-raw omit raw requests/responses from jsonl output -ob, -omit-body omit response body from jsonl output -j, -jsonl write output in jsonl format @@ -688,7 +689,7 @@ katana -u https://tesla.com -f email,phone *`-store-field`* --- -To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. +To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. Use `-sfd` or `-store-field-dir` to store data in a different location. ``` katana -u https://tesla.com -sf key,fqdn,qurl -silent diff --git a/cmd/katana/main.go b/cmd/katana/main.go index d6ce829f..338b4068 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -177,6 +177,7 @@ pipelines offering both headless and non-headless crawling.`) flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"), + flagSet.StringVarP(&options.StoreFieldDir, "store-field-dir", "sfd", "", "store per-host field to custom directory"), flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"), flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"), flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"), diff --git a/pkg/output/fields.go b/pkg/output/fields.go index ab8666b7..cf074a83 100644 --- a/pkg/output/fields.go +++ b/pkg/output/fields.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "path" - "path/filepath" "strings" "github.com/projectdiscovery/gologger" @@ -85,7 +84,7 @@ func storeFields(output *Result, storeFields []string) { } func appendToFileField(parsed *url.URL, field, data string) { - file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + file, err := os.OpenFile(path.Join(storeFieldDir, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { return } diff --git a/pkg/output/options.go b/pkg/output/options.go index d519c353..99faa32a 100644 --- a/pkg/output/options.go +++ b/pkg/output/options.go @@ -19,6 +19,7 @@ type Options struct { Fields string StoreFields string StoreResponseDir string + StoreFieldDir string FieldConfig string ErrorLogFile string MatchRegex []*regexp.Regexp diff --git a/pkg/output/output.go b/pkg/output/output.go index 90cb65ee..0e0e1983 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -21,12 +21,12 @@ import ( ) const ( - storeFieldsDirectory = "katana_field" - indexFile = "index.txt" - DefaultResponseDir = "katana_response" + indexFile = "index.txt" + DefaultResponseDir = "katana_response" ) var ( + storeFieldDir = "katana_field" decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) ) @@ -80,6 +80,10 @@ func New(options Options) (Writer, error) { outputMatchCondition: options.OutputMatchCondition, outputFilterCondition: options.OutputFilterCondition, } + + if options.StoreFieldDir != "" { + storeFieldDir = options.StoreFieldDir + } // if fieldConfig empty get the default file if options.FieldConfig == "" { var err error @@ -103,7 +107,7 @@ func New(options Options) (Writer, error) { } } if options.StoreFields != "" { - _ = os.MkdirAll(storeFieldsDirectory, os.ModePerm) + _ = os.MkdirAll(storeFieldDir, os.ModePerm) if err := validateFieldNames(options.StoreFields); err != nil { return nil, errorutil.NewWithTag("output", "could not validate store fields").Wrap(err) } diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index 05dc4c9d..86cb6592 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -70,6 +70,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { StoreFields: options.StoreFields, StoreResponseDir: options.StoreResponseDir, NoClobber: options.NoClobber, + StoreFieldDir: options.StoreFieldDir, OmitRaw: options.OmitRaw, OmitBody: options.OmitBody, FieldConfig: options.FieldConfig, diff --git a/pkg/types/options.go b/pkg/types/options.go index b402d628..6b8eefb1 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -119,6 +119,8 @@ type Options struct { StoreResponseDir string // NoClobber specifies if katana should overwrite existing output files NoClobber bool + // StoreFieldDir specifies if katana should use a custom directory to store fields + StoreFieldDir string // OmitRaw omits raw requests/responses from the output OmitRaw bool // OmitBody omits the response body from the output From 4472681b0b2ae60260acda584097cdf027d57d48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:09:19 +0000 Subject: [PATCH 099/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.65 to 1.0.67. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.65...v1.0.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 950ecba5..3ce781f4 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,10 @@ require ( github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.57 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.47 + github.com/projectdiscovery/hmap v0.0.48 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 - github.com/projectdiscovery/retryablehttp-go v1.0.65 + github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.57 github.com/projectdiscovery/utils v0.1.5 github.com/projectdiscovery/wappalyzergo v0.1.8 @@ -126,7 +126,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/retryabledns v1.0.64 // indirect + github.com/projectdiscovery/retryabledns v1.0.65 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 2b22615f..53904b40 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk= -github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg= +github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo= +github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= @@ -231,10 +231,10 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= -github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= -github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= -github.com/projectdiscovery/retryablehttp-go v1.0.65 h1:GCZI9CUbYkldy9Iup4C7w6aVTxZuAHpSZTVy92qHPo4= -github.com/projectdiscovery/retryablehttp-go v1.0.65/go.mod h1:qF0UuglJiYJcezEoMIyYVYhHGwaOBy8bh3gRGz7CBs8= +github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ= +github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= +github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= +github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= From 057cc3702ad63777c697cdc5b3fed99699ed567f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:09:24 +0000 Subject: [PATCH 100/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.1 to 0.1.4 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.1 to 0.1.4. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.1...v0.1.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 950ecba5..b3f49c9c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.1 + github.com/projectdiscovery/dsl v0.1.4 github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.57 github.com/projectdiscovery/gologger v1.1.12 diff --git a/go.sum b/go.sum index 2b22615f..ea2f54e0 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= -github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= +github.com/projectdiscovery/dsl v0.1.4 h1:tWF2xWmfwrO/UhalruDflQEBr19WBwU9My4Uke4jndQ= +github.com/projectdiscovery/dsl v0.1.4/go.mod h1:jYwRd4bRhCtWAOEsG347W4pCi9c/fKfm7vtmuFhwYtA= github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= From 18b36726b7ff375dd893ef3c94c9471eacf0efd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:40:49 +0000 Subject: [PATCH 101/183] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.12 to 1.1.13. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.12...v1.1.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bc6e50ea..deae80d3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/projectdiscovery/dsl v0.1.4 github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.57 - github.com/projectdiscovery/gologger v1.1.12 + github.com/projectdiscovery/gologger v1.1.13 github.com/projectdiscovery/hmap v0.0.48 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.45 diff --git a/go.sum b/go.sum index f6a92c41..6d80c0a8 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNY github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= -github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= -github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E= +github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo= From 8a104109744efea9a3f248ae8393f355c6361aa9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:54:07 +0000 Subject: [PATCH 102/183] chore(deps): bump github.com/projectdiscovery/utils from 0.1.5 to 0.2.0 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.1.5 to 0.2.0. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.1.5...v0.2.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index deae80d3..67d8df8f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.57 - github.com/projectdiscovery/utils v0.1.5 + github.com/projectdiscovery/utils v0.2.0 github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 6d80c0a8..0b4c2db9 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJN github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= -github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= -github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= +github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= +github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 2862d9817c4e25fee8ec4230d2c66a8a69fd0e40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:22:26 +0000 Subject: [PATCH 103/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.45 to 0.0.46. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.45...v0.0.46) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 67d8df8f..685f7752 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.13 github.com/projectdiscovery/hmap v0.0.48 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.45 + github.com/projectdiscovery/ratelimit v0.0.46 github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.57 github.com/projectdiscovery/utils v0.2.0 diff --git a/go.sum b/go.sum index 0b4c2db9..f047d3d0 100644 --- a/go.sum +++ b/go.sum @@ -229,8 +229,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= -github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= +github.com/projectdiscovery/ratelimit v0.0.46 h1:asa5DpyLhMIbx4x6Y07RXycQrRcZCdacehkxqtAMksU= +github.com/projectdiscovery/ratelimit v0.0.46/go.mod h1:zjnw0ONxJysJUp0gOWPziLq2PC5PN42Ldlm2+HJYr8Q= github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ= github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= From 06031367bce84939879145674dd344b5bc3b8f02 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:22:35 +0000 Subject: [PATCH 104/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.4 to 0.1.5 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.4 to 0.1.5. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.4...v0.1.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 67d8df8f..77f90f89 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.4 + github.com/projectdiscovery/dsl v0.1.5 github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.57 github.com/projectdiscovery/gologger v1.1.13 diff --git a/go.sum b/go.sum index 0b4c2db9..cfbcf654 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.4 h1:tWF2xWmfwrO/UhalruDflQEBr19WBwU9My4Uke4jndQ= -github.com/projectdiscovery/dsl v0.1.4/go.mod h1:jYwRd4bRhCtWAOEsG347W4pCi9c/fKfm7vtmuFhwYtA= +github.com/projectdiscovery/dsl v0.1.5 h1:bVBUsT7cV1zvO7/ovTTwk8wbiNnqPIzSKT2+HLz+yyQ= +github.com/projectdiscovery/dsl v0.1.5/go.mod h1:HmWimbP8CW+EFhIxhU7CRV6nNBcXQEIWyIqeRj9ew/Q= github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= From 326afb9029896f8571c59565881ed25a54036e88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:22:42 +0000 Subject: [PATCH 105/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.8 to 0.1.9. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.8...v0.1.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 67d8df8f..38eff83a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.57 github.com/projectdiscovery/utils v0.2.0 - github.com/projectdiscovery/wappalyzergo v0.1.8 + github.com/projectdiscovery/wappalyzergo v0.1.9 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 0b4c2db9..6913f606 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpu github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= -github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= -github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.9 h1:d92ZxGNxnzmjTdIDAufQF87ghsxp6GqA3AVuJ8TpC/Q= +github.com/projectdiscovery/wappalyzergo v0.1.9/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From 7b5cad732183a74981f9d9d0f3cb9212bbc536cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:53:52 +0000 Subject: [PATCH 106/183] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.57 to 0.0.58. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.57...v0.0.58) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 76a83258..12673881 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.46 github.com/projectdiscovery/retryablehttp-go v1.0.67 - github.com/projectdiscovery/useragent v0.0.57 + github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.0 github.com/projectdiscovery/wappalyzergo v0.1.9 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 8c263ff5..a35e807c 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,8 @@ github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJOD github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI= -github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= -github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= +github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= +github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= github.com/projectdiscovery/wappalyzergo v0.1.9 h1:d92ZxGNxnzmjTdIDAufQF87ghsxp6GqA3AVuJ8TpC/Q= From 012edda3a34eea73f549618c71941a166c3591f1 Mon Sep 17 00:00:00 2001 From: Federico Muttis Date: Sat, 13 Jul 2024 17:51:22 +0200 Subject: [PATCH 107/183] When there is no value on an input, favor HTML's placeholder before using Katana's if exists --- pkg/utils/formfill.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 1ace9a35..2deeb742 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -47,7 +47,7 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st data := mapsutil.NewOrderedMap[string, string]() // Fill checkboxes and radioboxes first or default values first - for _, input := range inputs { + for i, input := range inputs { switch input.Type { case "radio": // Use a single radio name per value @@ -62,6 +62,11 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st // infer the values based on input types. if input.Value != "" { data.Set(input.Name, input.Value) + } else { + if value, ok := input.Attributes.Get("placeholder"); ok { + inputs[i].Value = value + data.Set(input.Name, value) + } } } } From 6113a812a0407c2e2f8b770506d4abbd0ebe0b0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:53:59 +0000 Subject: [PATCH 108/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.9 to 0.1.10. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.9...v0.1.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 12673881..5da5f093 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.0 - github.com/projectdiscovery/wappalyzergo v0.1.9 + github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index a35e807c..11c28530 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyD github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= -github.com/projectdiscovery/wappalyzergo v0.1.9 h1:d92ZxGNxnzmjTdIDAufQF87ghsxp6GqA3AVuJ8TpC/Q= -github.com/projectdiscovery/wappalyzergo v0.1.9/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= +github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= From ed81328a21c76d4ee7c34cddc3ba298495a99bcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:54:04 +0000 Subject: [PATCH 109/183] chore(deps): bump github.com/projectdiscovery/asnmap from 1.1.0 to 1.1.1 Bumps [github.com/projectdiscovery/asnmap](https://github.com/projectdiscovery/asnmap) from 1.1.0 to 1.1.1. - [Release notes](https://github.com/projectdiscovery/asnmap/releases) - [Changelog](https://github.com/projectdiscovery/asnmap/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/asnmap/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/asnmap dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 12673881..94affbcd 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/muesli/termenv v0.15.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect - github.com/projectdiscovery/asnmap v1.1.0 // indirect + github.com/projectdiscovery/asnmap v1.1.1 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect diff --git a/go.sum b/go.sum index a35e807c..4e7eecfd 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6KLAW13m0= -github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= +github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= +github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.5 h1:bVBUsT7cV1zvO7/ovTTwk8wbiNnqPIzSKT2+HLz+yyQ= From 1c911432d4f523fea39c76463f4c57e72db1a2c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:54:14 +0000 Subject: [PATCH 110/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.46 to 0.0.47. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.46...v0.0.47) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 12673881..42ecae4e 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.13 github.com/projectdiscovery/hmap v0.0.48 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.46 + github.com/projectdiscovery/ratelimit v0.0.47 github.com/projectdiscovery/retryablehttp-go v1.0.67 github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.0 diff --git a/go.sum b/go.sum index a35e807c..e110d714 100644 --- a/go.sum +++ b/go.sum @@ -229,8 +229,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.46 h1:asa5DpyLhMIbx4x6Y07RXycQrRcZCdacehkxqtAMksU= -github.com/projectdiscovery/ratelimit v0.0.46/go.mod h1:zjnw0ONxJysJUp0gOWPziLq2PC5PN42Ldlm2+HJYr8Q= +github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0fQmwv/Y0epQAM= +github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ= github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= From d9667a50fd459fea7e063481b2161b5d77aba972 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:25:18 +0000 Subject: [PATCH 111/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.5 to 0.1.6 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.5 to 0.1.6. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.5...v0.1.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index dd6eb8e8..c21c7a78 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,10 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.5 + github.com/projectdiscovery/dsl v0.1.6 github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/goflags v0.1.57 - github.com/projectdiscovery/gologger v1.1.13 + github.com/projectdiscovery/gologger v1.1.14 github.com/projectdiscovery/hmap v0.0.48 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 diff --git a/go.sum b/go.sum index 71c96a50..bd74e060 100644 --- a/go.sum +++ b/go.sum @@ -211,14 +211,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.5 h1:bVBUsT7cV1zvO7/ovTTwk8wbiNnqPIzSKT2+HLz+yyQ= -github.com/projectdiscovery/dsl v0.1.5/go.mod h1:HmWimbP8CW+EFhIxhU7CRV6nNBcXQEIWyIqeRj9ew/Q= +github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= +github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= -github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E= -github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI= +github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= +github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo= From 3d66ef04e1c7c59c9869688b27d54c1d142c003c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:25:37 +0000 Subject: [PATCH 112/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.65 to 1.0.67. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.65...v1.0.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd6eb8e8..c134f525 100644 --- a/go.mod +++ b/go.mod @@ -126,7 +126,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/retryabledns v1.0.65 // indirect + github.com/projectdiscovery/retryabledns v1.0.67 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 71c96a50..226ab83a 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0fQmwv/Y0epQAM= github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= -github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ= -github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= +github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= +github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= From bee95ef12ce562934afa28ef57bd7db3083b3e78 Mon Sep 17 00:00:00 2001 From: "alban.stourbe stourbe" Date: Tue, 16 Jul 2024 09:52:19 +0200 Subject: [PATCH 113/183] Add a small Merge Func for formill and enhances code based on the review --- pkg/utils/formfill.go | 44 +++++++++++++++++------------------------- pkg/utils/maps.go | 10 ++++++++++ pkg/utils/maps_test.go | 32 ++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 26 deletions(-) create mode 100644 pkg/utils/maps.go create mode 100644 pkg/utils/maps_test.go diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 57f4e56d..9d99002d 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -41,8 +41,8 @@ type FormInput struct { Attributes mapsutil.OrderedMap[string, string] } -// FormOption is an option for a select input -type FormOption struct { +// SelectOption is an option for a select input +type SelectOption struct { Value string Selected string Attributes mapsutil.OrderedMap[string, string] @@ -50,9 +50,9 @@ type FormOption struct { // FormSelect is a select input for a form field type FormSelect struct { - Name string - Attributes mapsutil.OrderedMap[string, string] - FormOptions []FormOption + Name string + Attributes mapsutil.OrderedMap[string, string] + SelectOptions []SelectOption } type FormTextArea struct { @@ -133,7 +133,7 @@ func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, st func FormSelectFill(inputs []FormSelect) mapsutil.OrderedMap[string, string] { data := mapsutil.NewOrderedMap[string, string]() for _, input := range inputs { - for _, option := range input.FormOptions { + for _, option := range input.SelectOptions { if option.Selected != "" { data.Set(input.Name, option.Value) break @@ -141,8 +141,8 @@ func FormSelectFill(inputs []FormSelect) mapsutil.OrderedMap[string, string] { } // If no option is selected, select the first one - if !data.Has(input.Name) && len(input.FormOptions) > 0 { - data.Set(input.Name, input.FormOptions[0].Value) + if !data.Has(input.Name) && len(input.SelectOptions) > 0 { + data.Set(input.Name, input.SelectOptions[0].Value) } } return data @@ -176,22 +176,14 @@ func FormFillSuggestions(formFields []interface{}) mapsutil.OrderedMap[string, s switch v := item.(type) { case FormInput: dataMapInputs := FormInputFillSuggestions([]FormInput{v}) - dataMapInputs.Iterate(func(key, value string) bool { - merged.Set(key, value) - return true - }) + MergeDataMaps(&merged, dataMapInputs) case FormSelect: dataMapSelects := FormSelectFill([]FormSelect{v}) - dataMapSelects.Iterate(func(key, value string) bool { - merged.Set(key, value) - return true - }) + MergeDataMaps(&merged, dataMapSelects) + case FormTextArea: dataMapTextArea := FormTextAreaFill([]FormTextArea{v}) - dataMapTextArea.Iterate(func(key, value string) bool { - merged.Set(key, value) - return true - }) + MergeDataMaps(&merged, dataMapTextArea) } } return merged @@ -217,11 +209,11 @@ func ConvertGoquerySelectionToFormInput(item *goquery.Selection) FormInput { return input } -// ConvertGoquerySelectionToFormOption converts a goquery.Selection object to a FormOption object. -// It extracts the attributes from the goquery.Selection object and populates a FormOption object with the extracted values. -func ConvertGoquerySelectionToFormOption(item *goquery.Selection) FormOption { +// ConvertGoquerySelectionToSelectOption converts a goquery.Selection object to a SelectOption object. +// It extracts the attributes from the goquery.Selection object and populates a SelectOption object with the extracted values. +func ConvertGoquerySelectionToSelectOption(item *goquery.Selection) SelectOption { attrs := item.Nodes[0].Attr - input := FormOption{Attributes: mapsutil.NewOrderedMap[string, string]()} + input := SelectOption{Attributes: mapsutil.NewOrderedMap[string, string]()} for _, attribute := range attrs { switch attribute.Key { case "value": @@ -251,9 +243,9 @@ func ConvertGoquerySelectionToFormSelect(item *goquery.Selection) FormSelect { } } - input.FormOptions = []FormOption{} + input.SelectOptions = []SelectOption{} item.Find("option").Each(func(_ int, option *goquery.Selection) { - input.FormOptions = append(input.FormOptions, ConvertGoquerySelectionToFormOption(option)) + input.SelectOptions = append(input.SelectOptions, ConvertGoquerySelectionToSelectOption(option)) }) return input } diff --git a/pkg/utils/maps.go b/pkg/utils/maps.go new file mode 100644 index 00000000..05cbe2b9 --- /dev/null +++ b/pkg/utils/maps.go @@ -0,0 +1,10 @@ +package utils + +import mapsutil "github.com/projectdiscovery/utils/maps" + +func MergeDataMaps(dataMap1 *mapsutil.OrderedMap[string, string], dataMap2 mapsutil.OrderedMap[string, string]) { + dataMap2.Iterate(func(key, value string) bool { + dataMap1.Set(key, value) + return true + }) +} diff --git a/pkg/utils/maps_test.go b/pkg/utils/maps_test.go new file mode 100644 index 00000000..2b1c37bc --- /dev/null +++ b/pkg/utils/maps_test.go @@ -0,0 +1,32 @@ +package utils + +import ( + "testing" + + mapsutil "github.com/projectdiscovery/utils/maps" + "github.com/stretchr/testify/require" +) + +func TestMergeDataMap(t *testing.T) { + // Create two data maps + dataMap1 := mapsutil.NewOrderedMap[string, string]() + dataMap1.Set("key1", "value1") + dataMap1.Set("key2", "value2") + + dataMap2 := mapsutil.NewOrderedMap[string, string]() + dataMap2.Set("key3", "value3") + dataMap2.Set("key4", "value4") + + // Merge the data maps + MergeDataMaps(&dataMap1, dataMap2) + + // Verify the merged map contains all the keys and values + value1, _ := dataMap1.Get("key1") + value2, _ := dataMap1.Get("key2") + value3, _ := dataMap1.Get("key3") + value4, _ := dataMap1.Get("key4") + require.Equal(t, "value1", value1) + require.Equal(t, "value2", value2) + require.Equal(t, "value3", value3) + require.Equal(t, "value4", value4) +} From 2a0f23c2ecd31df8e4848ca8603f503e946e0893 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:08:54 +0000 Subject: [PATCH 114/183] chore(deps): bump github.com/projectdiscovery/utils from 0.2.0 to 0.2.2 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.0 to 0.2.2. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.0...v0.2.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 16 +++++++--------- go.sum | 40 ++++++++++++++-------------------------- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index efe0dde4..940308d1 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.6 - github.com/projectdiscovery/fastdialer v0.1.5 + github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/goflags v0.1.57 - github.com/projectdiscovery/gologger v1.1.14 - github.com/projectdiscovery/hmap v0.0.48 + github.com/projectdiscovery/gologger v1.1.15 + github.com/projectdiscovery/hmap v0.0.50 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 - github.com/projectdiscovery/retryablehttp-go v1.0.67 + github.com/projectdiscovery/retryablehttp-go v1.0.69 github.com/projectdiscovery/useragent v0.0.58 - github.com/projectdiscovery/utils v0.2.0 + github.com/projectdiscovery/utils v0.2.2 github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -48,7 +48,6 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.9.5 // indirect - github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -57,7 +56,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/kataras/jwt v0.1.8 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -74,8 +73,7 @@ require ( github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/refraction-networking/utls v1.5.4 // indirect + github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sashabaranov/go-openai v1.14.2 // indirect diff --git a/go.sum b/go.sum index 9797e5d5..8d392a9c 100644 --- a/go.sum +++ b/go.sum @@ -68,16 +68,10 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= -github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= -github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.114.1 h1:osBWr88guzTXAIzwJWVmGZe3/utT9+lqKjkGSBsYMxw= github.com/go-rod/rod v0.114.1/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -104,8 +98,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= @@ -127,8 +119,8 @@ github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -192,8 +184,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= @@ -213,16 +203,16 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= -github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= -github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= +github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= +github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= -github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= -github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= +github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLiGyl203nDYbU= +github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo= -github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I= +github.com/projectdiscovery/hmap v0.0.50 h1:ZFY1JF6y6BeBoEYXi7h0/y/LS+nxN1NUo+KsyqCSuJM= +github.com/projectdiscovery/hmap v0.0.50/go.mod h1:cu4xZ/ZkntYI5NH7/ity6kK15XyjUF3fp78qb4shUxo= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= @@ -233,18 +223,16 @@ github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0f github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= -github.com/projectdiscovery/retryablehttp-go v1.0.67 h1:3jCiREsQEWJN2fG3D+i46zJNkq5+GtrLGf+Q77jGgrE= -github.com/projectdiscovery/retryablehttp-go v1.0.67/go.mod h1:LXTRQdTOOmMLJDbw5R9fyYgQKLjwS/l5tVt2wP09FBI= +github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= +github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= -github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= -github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= +github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= +github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= -github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= -github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= +github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= +github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From b37235595dca85dd002a56c23667d1741ebe6df0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:37:53 +0000 Subject: [PATCH 115/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.67 to 1.0.68. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.67...v1.0.68) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 940308d1..4e52cc3d 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/retryabledns v1.0.67 // indirect + github.com/projectdiscovery/retryabledns v1.0.68 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 8d392a9c..5ad015da 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0fQmwv/Y0epQAM= github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= -github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= -github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= +github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA= +github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg= github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= From 0e187dda32e4949c2fa6b9e89f538f9bf24cff9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:37:57 +0000 Subject: [PATCH 116/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.57 to 0.1.60. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.57...v0.1.60) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 940308d1..58da08cf 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.6 github.com/projectdiscovery/fastdialer v0.2.0 - github.com/projectdiscovery/goflags v0.1.57 + github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.15 github.com/projectdiscovery/hmap v0.0.50 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index 8d392a9c..3153f166 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8S github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= -github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= -github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= +github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= +github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLiGyl203nDYbU= github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From cc1ddae0acca9834302356754b3253b4e216222d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 13:07:11 +0000 Subject: [PATCH 117/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.6 to 0.1.7 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.6 to 0.1.7. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.6...v0.1.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d83722cb..f48ca98e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.6 + github.com/projectdiscovery/dsl v0.1.7 github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.15 diff --git a/go.sum b/go.sum index f6579169..bfc27eba 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= -github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= +github.com/projectdiscovery/dsl v0.1.7 h1:mPaHFPr2IMq69SjFGTt/DP8W5QeaGo+6lHqusnko2Qw= +github.com/projectdiscovery/dsl v0.1.7/go.mod h1:pjBGO539itAgO+qD5PSXTcvRiF92tJKLzkA1aqCMwEQ= github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= From e39dc891c4f25f78e3c580f24a57dacd25d3e903 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:26:40 +0000 Subject: [PATCH 118/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.68 to 1.0.69. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.68...v1.0.69) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f48ca98e..b010a87a 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/retryabledns v1.0.68 // indirect + github.com/projectdiscovery/retryabledns v1.0.69 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index bfc27eba..44f605d8 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0fQmwv/Y0epQAM= github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= -github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA= -github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg= +github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= +github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= From ccb33a2511f9fa74317b0b098ac7b78e8f22db42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:26:50 +0000 Subject: [PATCH 119/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.50 to 0.0.52 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.50 to 0.0.52. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.50...v0.0.52) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f48ca98e..cc0f9d2b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.15 - github.com/projectdiscovery/hmap v0.0.50 + github.com/projectdiscovery/hmap v0.0.52 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 github.com/projectdiscovery/retryablehttp-go v1.0.69 diff --git a/go.sum b/go.sum index bfc27eba..ca06a0cd 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLi github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.50 h1:ZFY1JF6y6BeBoEYXi7h0/y/LS+nxN1NUo+KsyqCSuJM= -github.com/projectdiscovery/hmap v0.0.50/go.mod h1:cu4xZ/ZkntYI5NH7/ity6kK15XyjUF3fp78qb4shUxo= +github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI= +github.com/projectdiscovery/hmap v0.0.52/go.mod h1:1YAS+8xYRIm+M8Qnlim8vw+U4rOPGLw5CqlirAVIPQ4= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From ea6cb8283cda8c2dbb2ca10915e3318b74228b44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:57:18 +0000 Subject: [PATCH 120/183] chore(deps): bump github.com/projectdiscovery/utils from 0.2.2 to 0.2.3 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.2 to 0.2.3. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.2...v0.2.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 2d8f0e81..252f6a4e 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.7 - github.com/projectdiscovery/fastdialer v0.2.0 + github.com/projectdiscovery/fastdialer v0.2.1 github.com/projectdiscovery/goflags v0.1.60 - github.com/projectdiscovery/gologger v1.1.15 + github.com/projectdiscovery/gologger v1.1.16 github.com/projectdiscovery/hmap v0.0.52 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 - github.com/projectdiscovery/retryablehttp-go v1.0.69 + github.com/projectdiscovery/retryablehttp-go v1.0.70 github.com/projectdiscovery/useragent v0.0.58 - github.com/projectdiscovery/utils v0.2.2 + github.com/projectdiscovery/utils v0.2.3 github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index d67a46c7..76c8d83d 100644 --- a/go.sum +++ b/go.sum @@ -203,12 +203,12 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.7 h1:mPaHFPr2IMq69SjFGTt/DP8W5QeaGo+6lHqusnko2Qw= github.com/projectdiscovery/dsl v0.1.7/go.mod h1:pjBGO539itAgO+qD5PSXTcvRiF92tJKLzkA1aqCMwEQ= -github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= -github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= +github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q= +github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= -github.com/projectdiscovery/gologger v1.1.15 h1:EgXC9uQkYb5qWcazV6mH0mev23MZ0GLiGyl203nDYbU= -github.com/projectdiscovery/gologger v1.1.15/go.mod h1:qPDCXZIxqlaKYhX0iyXZtsbtTFU152knbNRF4Z6fXUQ= +github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc= +github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI= @@ -223,12 +223,12 @@ github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0f github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= -github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= -github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= +github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0tGMTWBs80aFUZlICnRI= +github.com/projectdiscovery/retryablehttp-go v1.0.70/go.mod h1:54vRm5DSwGBbBXfsjKbFDXrr7JLefWkp0iBV9mbhdoA= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= -github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= -github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= +github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY= +github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c= github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= From 524849382eddae4b079638f3bd5163c36ef691cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:25:36 +0000 Subject: [PATCH 121/183] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.15 to 1.1.17. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.15...v1.1.17) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 252f6a4e..cd2968c7 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/projectdiscovery/dsl v0.1.7 github.com/projectdiscovery/fastdialer v0.2.1 github.com/projectdiscovery/goflags v0.1.60 - github.com/projectdiscovery/gologger v1.1.16 + github.com/projectdiscovery/gologger v1.1.17 github.com/projectdiscovery/hmap v0.0.52 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 diff --git a/go.sum b/go.sum index 76c8d83d..90c86e30 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,8 @@ github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+ya github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= -github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc= -github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= +github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7++166xDRSqo= +github.com/projectdiscovery/gologger v1.1.17/go.mod h1:JAo10tJopMAUHDdKTX/ZSb1QC/sgxfr5ovrd+SAt7fM= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI= From 3d83a7f7ff59c417047bcbea88ad5e23b6971eba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:55:28 +0000 Subject: [PATCH 122/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.7 to 0.1.8 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.7 to 0.1.8. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.7...v0.1.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cd2968c7..10462b22 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.7 + github.com/projectdiscovery/dsl v0.1.8 github.com/projectdiscovery/fastdialer v0.2.1 github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.17 diff --git a/go.sum b/go.sum index 90c86e30..3c58e728 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.7 h1:mPaHFPr2IMq69SjFGTt/DP8W5QeaGo+6lHqusnko2Qw= -github.com/projectdiscovery/dsl v0.1.7/go.mod h1:pjBGO539itAgO+qD5PSXTcvRiF92tJKLzkA1aqCMwEQ= +github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= +github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q= github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= From dd2e183eaeff42a846ca3f3db2080ebdffce2d11 Mon Sep 17 00:00:00 2001 From: 4shen0ne <33086594+zrquan@users.noreply.github.com> Date: Tue, 30 Jul 2024 19:19:35 +0800 Subject: [PATCH 123/183] no need for browser.close() (#972) When the katana process exits, rod will automatically kill the browser process. --- pkg/engine/hybrid/hybrid.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index bc9e3e5e..3e4e422b 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -96,11 +96,6 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { // Close closes the crawler process func (c *Crawler) Close() error { - if c.Options.Options.ChromeWSUrl == "" { - if err := c.browser.Close(); err != nil { - return err - } - } if c.Options.Options.ChromeDataDir == "" { if err := os.RemoveAll(c.tempDir); err != nil { return err From cdc224e881fc69450bf50b6705057b8d839be967 Mon Sep 17 00:00:00 2001 From: 4shen0ne <4shen.01@gmail.com> Date: Wed, 31 Jul 2024 19:31:13 +0800 Subject: [PATCH 124/183] ignore default paths if -scp is provided --- pkg/engine/hybrid/hybrid.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 3e4e422b..68852bc1 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -137,10 +137,14 @@ func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*laun UserDataDir(dataStore) if options.Options.UseInstalledChrome { - if chromePath, hasChrome := launcher.LookPath(); hasChrome { - chromeLauncher.Bin(chromePath) + if options.Options.SystemChromePath != "" { + chromeLauncher.Bin(options.Options.SystemChromePath) } else { - return nil, errorutil.NewWithTag("hybrid", "the chrome browser is not installed").WithLevel(errorutil.Fatal) + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errorutil.NewWithTag("hybrid", "the chrome browser is not installed").WithLevel(errorutil.Fatal) + } } } if options.Options.SystemChromePath != "" { From 982313d9f695e92dba2ee5efb73f6e7154d99d43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:43:28 +0000 Subject: [PATCH 125/183] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.52 to 0.0.53 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.52 to 0.0.53. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.52...v0.0.53) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10462b22..86af950b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/projectdiscovery/fastdialer v0.2.1 github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.17 - github.com/projectdiscovery/hmap v0.0.52 + github.com/projectdiscovery/hmap v0.0.53 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.47 github.com/projectdiscovery/retryablehttp-go v1.0.70 diff --git a/go.sum b/go.sum index 3c58e728..69bdfa43 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7 github.com/projectdiscovery/gologger v1.1.17/go.mod h1:JAo10tJopMAUHDdKTX/ZSb1QC/sgxfr5ovrd+SAt7fM= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.52 h1:Gm0aI0HDgsPW1+pq3PflO7PJ5cbOXUKt0DorlVsN3pI= -github.com/projectdiscovery/hmap v0.0.52/go.mod h1:1YAS+8xYRIm+M8Qnlim8vw+U4rOPGLw5CqlirAVIPQ4= +github.com/projectdiscovery/hmap v0.0.53 h1:V4/svLcNCIpAnhnjrZf8EH1bCfIpV4LYFIMklxk/fw4= +github.com/projectdiscovery/hmap v0.0.53/go.mod h1:5FrB92frAAhKQGnu5x1wy5gCYUAvutyxnSt0LsGOIKI= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From 30941e5af2cc5d2e34af57689cfe452f6307bdd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:43:32 +0000 Subject: [PATCH 126/183] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.60 to 0.1.62. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.60...v0.1.62) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10462b22..b2cc8a70 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.8 github.com/projectdiscovery/fastdialer v0.2.1 - github.com/projectdiscovery/goflags v0.1.60 + github.com/projectdiscovery/goflags v0.1.62 github.com/projectdiscovery/gologger v1.1.17 github.com/projectdiscovery/hmap v0.0.52 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index 3c58e728..c64cd0d3 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3C github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q= github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= -github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= -github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= +github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= +github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7++166xDRSqo= github.com/projectdiscovery/gologger v1.1.17/go.mod h1:JAo10tJopMAUHDdKTX/ZSb1QC/sgxfr5ovrd+SAt7fM= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From bb9d04b502a75651a23cafc9e5c6107483464f0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:43:38 +0000 Subject: [PATCH 127/183] chore(deps): bump github.com/projectdiscovery/utils from 0.2.3 to 0.2.4 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.3 to 0.2.4. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.3...v0.2.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10462b22..9175376f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.47 github.com/projectdiscovery/retryablehttp-go v1.0.70 github.com/projectdiscovery/useragent v0.0.58 - github.com/projectdiscovery/utils v0.2.3 + github.com/projectdiscovery/utils v0.2.4 github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 3c58e728..5b0d9e35 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0 github.com/projectdiscovery/retryablehttp-go v1.0.70/go.mod h1:54vRm5DSwGBbBXfsjKbFDXrr7JLefWkp0iBV9mbhdoA= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= -github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY= -github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c= +github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= +github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= From 68b6080f5701a0b9da2cc22dd34bbcb3a1a3a2db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:13:56 +0000 Subject: [PATCH 128/183] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.47 to 0.0.50. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.47...v0.0.50) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6bb79305..373deeef 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/gologger v1.1.17 github.com/projectdiscovery/hmap v0.0.53 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.47 + github.com/projectdiscovery/ratelimit v0.0.50 github.com/projectdiscovery/retryablehttp-go v1.0.70 github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.4 diff --git a/go.sum b/go.sum index 645084c1..eb1f7d07 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.47 h1:rDjZwUK8/KPx72jFiH2bW+b+QLDZx0fQmwv/Y0epQAM= -github.com/projectdiscovery/ratelimit v0.0.47/go.mod h1:cNtVx+Mlp4ZSPmd2Bf/g/kuHOQI2us3TgjCoaLLV/Ss= +github.com/projectdiscovery/ratelimit v0.0.50 h1:33uzpi9DLzhQEC8Erfo3LWSzvlVcpCwrKNlYaxclllo= +github.com/projectdiscovery/ratelimit v0.0.50/go.mod h1:FxoAGNVkZMXwETUd9NS7bVxm65HQrAya1yXiX+/Vuwk= github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0tGMTWBs80aFUZlICnRI= From 071aa3f3ca6e90771c550235467b307c8cdef5d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:44:44 +0000 Subject: [PATCH 129/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.70 to 1.0.72. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.70...v1.0.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 373deeef..5a798997 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,13 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.8 - github.com/projectdiscovery/fastdialer v0.2.1 + github.com/projectdiscovery/fastdialer v0.2.2 github.com/projectdiscovery/goflags v0.1.62 github.com/projectdiscovery/gologger v1.1.17 github.com/projectdiscovery/hmap v0.0.53 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.50 - github.com/projectdiscovery/retryablehttp-go v1.0.70 + github.com/projectdiscovery/retryablehttp-go v1.0.72 github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.4 github.com/projectdiscovery/wappalyzergo v0.1.10 diff --git a/go.sum b/go.sum index eb1f7d07..e0d7a0f1 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= -github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q= -github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= +github.com/projectdiscovery/fastdialer v0.2.2 h1:AiUT4jIbS6JyM2QBQH9f+ZFBgvOVHzQG5K+M2FCA5jA= +github.com/projectdiscovery/fastdialer v0.2.2/go.mod h1:QKrkTQiUhedqxqR65hLeN+FTbkul50TzMxgkk5Va+Uk= github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7++166xDRSqo= @@ -223,8 +223,8 @@ github.com/projectdiscovery/ratelimit v0.0.50 h1:33uzpi9DLzhQEC8Erfo3LWSzvlVcpCw github.com/projectdiscovery/ratelimit v0.0.50/go.mod h1:FxoAGNVkZMXwETUd9NS7bVxm65HQrAya1yXiX+/Vuwk= github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= -github.com/projectdiscovery/retryablehttp-go v1.0.70 h1:J19y9kM3SL9TlYRtMD1H8tx0tGMTWBs80aFUZlICnRI= -github.com/projectdiscovery/retryablehttp-go v1.0.70/go.mod h1:54vRm5DSwGBbBXfsjKbFDXrr7JLefWkp0iBV9mbhdoA= +github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= +github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= From a81fe45cbfe49bf84a15718f940083f9bcbbabb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:46:02 +0000 Subject: [PATCH 130/183] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.8 to 0.1.10 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.8 to 0.1.10. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.8...v0.1.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 7 +++---- go.sum | 10 ++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 5a798997..0c6fac87 100644 --- a/go.mod +++ b/go.mod @@ -11,15 +11,14 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.1.8 + github.com/projectdiscovery/dsl v0.1.10 github.com/projectdiscovery/fastdialer v0.2.2 github.com/projectdiscovery/goflags v0.1.62 - github.com/projectdiscovery/gologger v1.1.17 + github.com/projectdiscovery/gologger v1.1.18 github.com/projectdiscovery/hmap v0.0.53 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.50 github.com/projectdiscovery/retryablehttp-go v1.0.72 - github.com/projectdiscovery/useragent v0.0.58 github.com/projectdiscovery/utils v0.2.4 github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/remeh/sizedwaitgroup v1.0.0 @@ -140,7 +139,7 @@ require ( github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index e0d7a0f1..7a28a736 100644 --- a/go.sum +++ b/go.sum @@ -201,14 +201,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.1.8 h1:ulz+o097XsYgqP4QXaAQhVSkoeD2ZPWd29wX3CTodUA= -github.com/projectdiscovery/dsl v0.1.8/go.mod h1:AYJS2WQ/q0smr2v4pEJTg4DPe6k56KFKR7UFXvzNz/4= +github.com/projectdiscovery/dsl v0.1.10 h1:FBD5Muiwj3OdkDAYvmltQ0rcVMpLZK5IqNZi9Pd/gb0= +github.com/projectdiscovery/dsl v0.1.10/go.mod h1:Kg9snSkjTPZ2qiIDr4jjFI46FgJzvqz48qcHWUWY7lw= github.com/projectdiscovery/fastdialer v0.2.2 h1:AiUT4jIbS6JyM2QBQH9f+ZFBgvOVHzQG5K+M2FCA5jA= github.com/projectdiscovery/fastdialer v0.2.2/go.mod h1:QKrkTQiUhedqxqR65hLeN+FTbkul50TzMxgkk5Va+Uk= github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= -github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7++166xDRSqo= -github.com/projectdiscovery/gologger v1.1.17/go.mod h1:JAo10tJopMAUHDdKTX/ZSb1QC/sgxfr5ovrd+SAt7fM= +github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98= +github.com/projectdiscovery/gologger v1.1.18/go.mod h1:IgjKgVLfVGUovxlC6WJcXK0qvo/tNZ1P0+r6miQqyRk= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.53 h1:V4/svLcNCIpAnhnjrZf8EH1bCfIpV4LYFIMklxk/fw4= @@ -225,8 +225,6 @@ github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJN github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= -github.com/projectdiscovery/useragent v0.0.58 h1:dodSWkdVoZ36wgXuISXPHggornaOQyDN5H/DMYZYsQQ= -github.com/projectdiscovery/useragent v0.0.58/go.mod h1:jdbMuw4JbHITkOEMUGE2aPQYyB5LKWICdlg/QDtVZX4= github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= From 664ace356fb6a026afe3e8a4bd81d7cb38988b2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:17:02 +0000 Subject: [PATCH 131/183] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.69 to 1.0.71. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.69...v1.0.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c6fac87..23836a4d 100644 --- a/go.mod +++ b/go.mod @@ -123,7 +123,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/retryabledns v1.0.69 // indirect + github.com/projectdiscovery/retryabledns v1.0.71 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect diff --git a/go.sum b/go.sum index 7a28a736..f0dd1bb5 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.50 h1:33uzpi9DLzhQEC8Erfo3LWSzvlVcpCwrKNlYaxclllo= github.com/projectdiscovery/ratelimit v0.0.50/go.mod h1:FxoAGNVkZMXwETUd9NS7bVxm65HQrAya1yXiX+/Vuwk= -github.com/projectdiscovery/retryabledns v1.0.69 h1:6vhhlfxPQoJI0j1enYSzyagyAZJNfGSg9TNloVsxEBQ= -github.com/projectdiscovery/retryabledns v1.0.69/go.mod h1:5SDuGtXgL4W1z/9V+xnmHFYg03Fi1Ud/F2kqlPlAgfE= +github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs= +github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU= github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= From b277723e0b0dd56ca79bf9850b4d97148170e68a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:17:04 +0000 Subject: [PATCH 132/183] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.10 to 0.1.14. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.10...v0.1.14) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c6fac87..11810e5c 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.50 github.com/projectdiscovery/retryablehttp-go v1.0.72 github.com/projectdiscovery/utils v0.2.4 - github.com/projectdiscovery/wappalyzergo v0.1.10 + github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 7a28a736..5fb0cf41 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnP github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= -github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= -github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= +github.com/projectdiscovery/wappalyzergo v0.1.14/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= From 4d86bf779807b5902c9d46e6eb1e0d491d6a1f89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:47:16 +0000 Subject: [PATCH 133/183] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.2.2 to 0.2.4. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.2...v0.2.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6e0b91a6..039a4aca 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/dsl v0.1.10 - github.com/projectdiscovery/fastdialer v0.2.2 + github.com/projectdiscovery/fastdialer v0.2.4 github.com/projectdiscovery/goflags v0.1.62 github.com/projectdiscovery/gologger v1.1.18 github.com/projectdiscovery/hmap v0.0.53 diff --git a/go.sum b/go.sum index f18fd70e..09db1bdd 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/dsl v0.1.10 h1:FBD5Muiwj3OdkDAYvmltQ0rcVMpLZK5IqNZi9Pd/gb0= github.com/projectdiscovery/dsl v0.1.10/go.mod h1:Kg9snSkjTPZ2qiIDr4jjFI46FgJzvqz48qcHWUWY7lw= -github.com/projectdiscovery/fastdialer v0.2.2 h1:AiUT4jIbS6JyM2QBQH9f+ZFBgvOVHzQG5K+M2FCA5jA= -github.com/projectdiscovery/fastdialer v0.2.2/go.mod h1:QKrkTQiUhedqxqR65hLeN+FTbkul50TzMxgkk5Va+Uk= +github.com/projectdiscovery/fastdialer v0.2.4 h1:tcAqaa93Bhp/w5uGjQXiPu4kpJurSeRi+YPIf0j8ii4= +github.com/projectdiscovery/fastdialer v0.2.4/go.mod h1:OrrHhUlXsFV6ZOOrg/ZS/HL2Oh/+wZziR8qrQv9fg28= github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= github.com/projectdiscovery/gologger v1.1.18 h1:fux2S/LXYe18/w4PylAFpEFg+EApNCuyS85x25nDE98= From 172627cfafe20c4822a75869d66c04a383b93353 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:15:51 +0000 Subject: [PATCH 134/183] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.72 to 1.0.73. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.72...v1.0.73) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 039a4aca..798bf4dc 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/hmap v0.0.53 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/ratelimit v0.0.50 - github.com/projectdiscovery/retryablehttp-go v1.0.72 + github.com/projectdiscovery/retryablehttp-go v1.0.73 github.com/projectdiscovery/utils v0.2.4 github.com/projectdiscovery/wappalyzergo v0.1.14 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 09db1bdd..a1b14b11 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/projectdiscovery/ratelimit v0.0.50 h1:33uzpi9DLzhQEC8Erfo3LWSzvlVcpCw github.com/projectdiscovery/ratelimit v0.0.50/go.mod h1:FxoAGNVkZMXwETUd9NS7bVxm65HQrAya1yXiX+/Vuwk= github.com/projectdiscovery/retryabledns v1.0.71 h1:hZnTR7C4G7XDLxQEvdZVREHEJToIfeObNwR+zpmZ1vs= github.com/projectdiscovery/retryabledns v1.0.71/go.mod h1:P3GAytlIf2c/H+PbHGwLoPrc8pSsCK+i8gt3sO/VynU= -github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= -github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= +github.com/projectdiscovery/retryablehttp-go v1.0.73 h1:RGckoCYNYrPbNtJRs7wy6vTFwckjZUZuPozUaCm51q8= +github.com/projectdiscovery/retryablehttp-go v1.0.73/go.mod h1:6ymJWVoTY05eB2nE/EqUcXgyFaZcBVPokUVLHz5JXi8= github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= github.com/projectdiscovery/wappalyzergo v0.1.14 h1:nt1IM4RUmqeymsXk4h6BsZbKDoS2hjFvPkT2GaI1rz4= From 6be6d72168667521f52a12ebd2e0eca3eb0b91df Mon Sep 17 00:00:00 2001 From: 4shen0ne <4shen.01@gmail.com> Date: Fri, 16 Aug 2024 11:16:37 +0800 Subject: [PATCH 135/183] add htmx parser --- pkg/engine/parser/parser.go | 43 ++++++++++++++++++++++++++++++++ pkg/engine/parser/parser_test.go | 34 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/pkg/engine/parser/parser.go b/pkg/engine/parser/parser.go index 4fa13fc3..c6b0b505 100644 --- a/pkg/engine/parser/parser.go +++ b/pkg/engine/parser/parser.go @@ -63,6 +63,7 @@ var responseParsers = []responseParser{ {bodyParser, bodyMetaContentTagParser}, {bodyParser, bodyHtmlManifestTagParser}, {bodyParser, bodyHtmlDoctypeTagParser}, + {bodyParser, bodyHtmxAttrParser}, // custom field regex parser {bodyParser, customFieldRegexParser}, @@ -600,6 +601,48 @@ func bodyMetaContentTagParser(resp *navigation.Response) (navigationRequests []* return } +func bodyHtmxAttrParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { + // exclude hx-delete + resp.Reader.Find("[hx-get],[hx-post],[hx-put],[hx-patch]").Each(func(i int, item *goquery.Selection) { + req := &navigation.Request{ + RootHostname: resp.RootHostname, + Depth: resp.Depth, + Source: resp.Resp.Request.URL.String(), + Tag: "htmx", + } + + hxGet, ok := item.Attr("hx-get") + if ok && hxGet != "" { + req.Method = "GET" + req.URL = resp.AbsoluteURL(hxGet) + req.Attribute = "hx-get" + navigationRequests = append(navigationRequests, req) + } + hxPost, ok := item.Attr(("hx-post")) + if ok && hxPost != "" { + req.Method = "POST" + req.URL = resp.AbsoluteURL(hxPost) + req.Attribute = "hx-post" + navigationRequests = append(navigationRequests, req) + } + hxPut, ok := item.Attr(("hx-put")) + if ok && hxPut != "" { + req.Method = "PUT" + req.URL = resp.AbsoluteURL(hxPut) + req.Attribute = "hx-put" + navigationRequests = append(navigationRequests, req) + } + hxPatch, ok := item.Attr(("hx-patch")) + if ok && hxPatch != "" { + req.Method = "PATCH" + req.URL = resp.AbsoluteURL(hxPatch) + req.Attribute = "hx-patch" + navigationRequests = append(navigationRequests, req) + } + }) + return +} + // ------------------------------------------------------------------------- // Begin JS Regex based parsers // ------------------------------------------------------------------------- diff --git a/pkg/engine/parser/parser_test.go b/pkg/engine/parser/parser_test.go index ac9a580f..a381c875 100644 --- a/pkg/engine/parser/parser_test.go +++ b/pkg/engine/parser/parser_test.go @@ -489,3 +489,37 @@ func TestRegexBodyParsers(t *testing.T) { require.Equal(t, requireFields, navigationRequests[0].CustomFields, "could not get correct url") }) } + +func TestHtmxBodyParser(t *testing.T) { + parsed, _ := urlutil.Parse("https://htmx.org/examples/") + + t.Run("hx-get", func(t *testing.T) { + documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(``)) + resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader} + navigationRequests := bodyHtmxAttrParser(resp) + require.Equal(t, "https://htmx.org/contact/1/edit", navigationRequests[0].URL, "could not get correct url") + require.Equal(t, "GET", navigationRequests[0].Method, "could not get correct method") + }) + t.Run("hx-post", func(t *testing.T) { + documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`
`)) + resp := &navigation.Response{Resp: &http.Response{Request: &http.Request{URL: parsed.URL}}, Reader: documentReader} + navigationRequests := bodyHtmxAttrParser(resp) + require.Equal(t, "https://htmx.org/users", navigationRequests[0].URL, "could not get correct url") + require.Equal(t, "POST", navigationRequests[0].Method, "could not get correct method") + }) + t.Run("hx-put", func(t *testing.T) { + documentReader, _ := goquery.NewDocumentFromReader(strings.NewReader(`