add -store-field-dir flag (#877)

README.md

@@ -193,6 +193,7 @@ OUTPUT:
    -o, -output string                file to write output to
    -sr, -store-response              store http requests/responses
    -srd, -store-response-dir string  store http requests/responses to custom directory
+   -sfd, -store-field-dir string     store per-host field to custom directory
    -or, -omit-raw                    omit raw requests/responses from jsonl output
    -ob, -omit-body                   omit response body from jsonl output
    -j, -jsonl                        write output in jsonl format
@@ -688,7 +689,7 @@ katana -u https://tesla.com -f email,phone
 *`-store-field`*
 ---
 
-To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url.
+To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. Use `-sfd` or `-store-field-dir` to store data in a different location.
 
 ```
 katana -u https://tesla.com -sf key,fqdn,qurl -silent

cmd/katana/main.go

@@ -177,6 +177,7 @@ pipelines offering both headless and non-headless crawling.`)
 		flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"),
 		flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"),
+		flagSet.StringVarP(&options.StoreFieldDir, "store-field-dir", "sfd", "", "store per-host field to custom directory"),
 		flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"),
 		flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"),
 		flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"),

pkg/output/fields.go

@@ -5,7 +5,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
@@ -85,7 +84,7 @@ func storeFields(output *Result, storeFields []string) {
 }
 
 func appendToFileField(parsed *url.URL, field, data string) {
-	file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	file, err := os.OpenFile(path.Join(storeFieldDir, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	if err != nil {
 		return
 	}

pkg/output/options.go

@@ -19,6 +19,7 @@ type Options struct {
 	Fields                string
 	StoreFields           string
 	StoreResponseDir      string
+	StoreFieldDir         string
 	FieldConfig           string
 	ErrorLogFile          string
 	MatchRegex            []*regexp.Regexp

pkg/output/output.go

@@ -21,12 +21,12 @@ import (
 )
 
 const (
-	storeFieldsDirectory = "katana_field"
-	indexFile            = "index.txt"
-	DefaultResponseDir   = "katana_response"
+	indexFile          = "index.txt"
+	DefaultResponseDir = "katana_response"
 )
 
 var (
+	storeFieldDir    = "katana_field"
 	decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
 )
 
@@ -80,6 +80,10 @@ func New(options Options) (Writer, error) {
 		outputMatchCondition:  options.OutputMatchCondition,
 		outputFilterCondition: options.OutputFilterCondition,
 	}
+
+	if options.StoreFieldDir != "" {
+		storeFieldDir = options.StoreFieldDir
+	}
 	// if fieldConfig empty get the default file
 	if options.FieldConfig == "" {
 		var err error
@@ -103,7 +107,7 @@ func New(options Options) (Writer, error) {
 		}
 	}
 	if options.StoreFields != "" {
-		_ = os.MkdirAll(storeFieldsDirectory, os.ModePerm)
+		_ = os.MkdirAll(storeFieldDir, os.ModePerm)
 		if err := validateFieldNames(options.StoreFields); err != nil {
 			return nil, errorutil.NewWithTag("output", "could not validate store fields").Wrap(err)
 		}

pkg/types/crawler_options.go

@@ -70,6 +70,7 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) {
 		StoreFields:           options.StoreFields,
 		StoreResponseDir:      options.StoreResponseDir,
 		NoClobber:             options.NoClobber,
+		StoreFieldDir:         options.StoreFieldDir,
 		OmitRaw:               options.OmitRaw,
 		OmitBody:              options.OmitBody,
 		FieldConfig:           options.FieldConfig,

pkg/types/options.go

@@ -119,6 +119,8 @@ type Options struct {
 	StoreResponseDir string
 	// NoClobber specifies if katana should overwrite existing output files
 	NoClobber bool
+	// StoreFieldDir specifies if katana should use a custom directory to store fields
+	StoreFieldDir string
 	// OmitRaw omits raw requests/responses from the output
 	OmitRaw bool
 	// OmitBody omits the response body from the output