Skip to content

Commit

Permalink
add no-clobber flag
Browse files Browse the repository at this point in the history
  • Loading branch information
@dogancanbakir
dogancanbakir committed Apr 18, 2024
1 parent 6d6db43 commit 26f87d8
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 2 deletions.
1 change: 1 addition & 0 deletions cmd/katana/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ pipelines offering both headless and non-headless crawling.`)
flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"),
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"),
flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"),
flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"),
flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"),
flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"),
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/dsl v0.0.52
github.com/projectdiscovery/fastdialer v0.0.67
github.com/projectdiscovery/fileutil v0.0.3
github.com/projectdiscovery/goflags v0.1.47
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/hmap v0.0.41
Expand Down
8 changes: 8 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
Expand Down Expand Up @@ -210,6 +211,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S
github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg=
github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
github.com/projectdiscovery/goflags v0.1.47 h1:cO0m+Xl4kXvwlyN4Yp61te+Utf3y2IiLXB1JRLvdFMY=
github.com/projectdiscovery/goflags v0.1.47/go.mod h1:+JKmFaqKtFtDYyhmfKEAk5BqnGRQuSF0fR+nk2r9oBQ=
github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
Expand Down Expand Up @@ -282,6 +285,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
Expand Down Expand Up @@ -402,6 +406,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
Expand Down Expand Up @@ -440,6 +445,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -452,6 +458,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
Expand All @@ -463,6 +470,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
Expand Down
1 change: 1 addition & 0 deletions pkg/output/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type Options struct {
JSON bool
Verbose bool
StoreResponse bool
NoClobber bool
OmitRaw bool
OmitBody bool
OutputFile string
Expand Down
60 changes: 58 additions & 2 deletions pkg/output/output.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import (
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"

jsoniter "github.com/json-iterator/go"
"github.com/logrusorgru/aurora"
"github.com/mitchellh/mapstructure"
"github.com/projectdiscovery/dsl"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/katana/pkg/utils/extensions"
errorutil "github.com/projectdiscovery/utils/errors"
Expand Down Expand Up @@ -48,6 +50,7 @@ type StandardWriter struct {
outputMutex *sync.Mutex
storeResponse bool
storeResponseDir string
noClobber bool
omitRaw bool
omitBody bool
errorFile *fileWriter
Expand All @@ -68,6 +71,7 @@ func New(options Options) (Writer, error) {
outputMutex: &sync.Mutex{},
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
noClobber: options.NoClobber,
omitRaw: options.OmitRaw,
omitBody: options.OmitBody,
matchRegex: options.MatchRegex,
Expand Down Expand Up @@ -117,8 +121,13 @@ func New(options Options) (Writer, error) {
if options.StoreResponseDir != DefaultResponseDir && options.StoreResponseDir != "" {
writer.storeResponseDir = options.StoreResponseDir
}
_ = os.RemoveAll(writer.storeResponseDir)
_ = os.MkdirAll(writer.storeResponseDir, os.ModePerm)
if options.NoClobber {
writer.storeResponseDir = createDirNameNoClobber(writer.storeResponseDir)
_ = os.MkdirAll(writer.storeResponseDir, os.ModePerm)
} else {
removeDirsWithSuffix(writer.storeResponseDir)
_ = os.MkdirAll(writer.storeResponseDir, os.ModePerm)
}
// todo: the index file seems never used?
_, err := newFileOutputWriter(filepath.Join(writer.storeResponseDir, indexFile))
if err != nil {
Expand Down Expand Up @@ -252,6 +261,53 @@ func (w *StandardWriter) Close() error {
return nil
}

func createDirNameNoClobber(dir string) string {
if !fileutil.FolderExists(dir) {
return dir
}

parentDir, dirName := filepath.Dir(dir), filepath.Base(dir)
entries, err := os.ReadDir(parentDir)
if err != nil {
return dirName
}

highestNum := 0
regex := regexp.MustCompile(fmt.Sprintf("^%s(\\d+)$", regexp.QuoteMeta(dirName)))
for _, entry := range entries {
if entry.IsDir() {
name := entry.Name()
matches := regex.FindStringSubmatch(name)
if matches != nil {
if num, err := strconv.Atoi(matches[1]); err == nil && num > highestNum {
highestNum = num
}
}
}
}

newDirName := fmt.Sprintf("%s%d", dirName, highestNum+1)
newFullPath := fmt.Sprintf("%s/%s", parentDir, newDirName)
return newFullPath
}

func removeDirsWithSuffix(dir string) {
parentDir, dirName := filepath.Dir(dir), filepath.Base(dir)
entries, _ := os.ReadDir(parentDir)

pattern := fmt.Sprintf("^%s(\\d*)$", regexp.QuoteMeta(dirName))
regex := regexp.MustCompile(pattern)
for _, entry := range entries {
if entry.IsDir() {
name := entry.Name()
if regex.MatchString(name) {
fullPath := filepath.Join(parentDir, name)
_ = os.RemoveAll(fullPath)
}
}
}
}

// matchOutput checks if the event matches the output regex
func (w *StandardWriter) matchOutput(event *Result) bool {
if w.matchRegex == nil && w.outputMatchCondition == "" {
Expand Down
1 change: 1 addition & 0 deletions pkg/types/crawler_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) {
Fields: options.Fields,
StoreFields: options.StoreFields,
StoreResponseDir: options.StoreResponseDir,
NoClobber: options.NoClobber,
OmitRaw: options.OmitRaw,
OmitBody: options.OmitBody,
FieldConfig: options.FieldConfig,
Expand Down
2 changes: 2 additions & 0 deletions pkg/types/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ type Options struct {
StoreResponse bool
// StoreResponseDir specifies if katana should use a custom directory to store http requests/responses
StoreResponseDir string
// NoClobber specifies if katana should overwrite existing output files
NoClobber bool
// OmitRaw omits raw requests/responses from the output
OmitRaw bool
// OmitBody omits the response body from the output
Expand Down

0 comments on commit 26f87d8

Please sign in to comment.