Added search to the api, and postgres support for DB

.gitignore

@@ -4,7 +4,7 @@
 *.so
 *.dylib
 *.sqlite3
-
+.idea
 # dep vendor/
 vendor/
 

chrome/chrome.go

@@ -3,6 +3,7 @@ package chrome
 import (
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"errors"
 	"io"
 	"net/http"
@@ -168,8 +169,8 @@ func (chrome *Chrome) StoreRequest(db *gorm.DB, preflight *PreflightResult, scre
 		Title:          preflight.HTTPTitle,
 		Filename:       filename,
 		IsPDF:          chrome.AsPDF,
+		Screenshot:     base64.StdEncoding.EncodeToString(screenshot.Screenshot),
 	}
-
 	// append headers
 	for k, v := range preflight.HTTPResponse.Header {
 		hv := strings.Join(v, ", ")

cmd/root.go

@@ -59,6 +59,7 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&options.DisableLogging, "disable-logging", false, "disable all logging")
 	// global
 	rootCmd.PersistentFlags().BoolVar(&db.Disabled, "disable-db", false, "disable all database operations")
+	rootCmd.PersistentFlags().IntVarP(&db.Platform, "type", "t", 0, "0 - SQLite, 1 - Postgres")
 	rootCmd.PersistentFlags().BoolVar(&db.Debug, "debug-db", false, "enable debug logging for all database operations")
 	rootCmd.PersistentFlags().StringVarP(&db.Path, "db-path", "D", "gowitness.sqlite3", "destination for the gowitness database")
 	rootCmd.PersistentFlags().IntVarP(&chrm.ResolutionX, "resolution-x", "X", 1440, "screenshot resolution x")
@@ -73,4 +74,5 @@ func init() {
 	rootCmd.PersistentFlags().Int64Var(&chrm.Timeout, "timeout", 10, "preflight check timeout")
 	rootCmd.PersistentFlags().StringVarP(&chrm.ChromePath, "chrome-path", "", "", "path to chrome executable to use")
 	rootCmd.PersistentFlags().StringVarP(&chrm.Proxy, "proxy", "p", "", "http/socks5 proxy to use. Use format proto://address:port")
+
 }

cmd/server.go

@@ -57,9 +57,17 @@ $ gowitness server --address 127.0.0.1:9000 --allow-insecure-uri`,
 		if !strings.Contains(options.ServerAddr, "localhost") {
 			log.Warn().Msg("exposing this server to other networks is dangerous! see the server command help for more information")
 		}
+		if db.Platform == storage.Sqlite {
+			if !strings.HasPrefix(options.BasePath, "/") {
+				log.Warn().Msg("base path does not start with a /")
+			}
+		}
+
+		if db.Platform == storage.Postgres {
+			if !strings.HasPrefix(options.BasePath, "postgresql") {
+				log.Warn().Msg("base path does not start with a /")
+			}
 
-		if !strings.HasPrefix(options.BasePath, "/") {
-			log.Warn().Msg("base path does not start with a /")
 		}
 
 		// db
@@ -125,6 +133,7 @@ $ gowitness server --address 127.0.0.1:9000 --allow-insecure-uri`,
 		api := r.Group("/api")
 		{
 			api.GET("/list", apiURLHandler)
+			api.GET("/search", apiSearchHandler)
 			api.GET("/detail/:id", apiDetailHandler)
 			api.GET("/detail/:id/screenshot", apiDetailScreenshotHandler)
 			api.POST("/screenshot", apiScreenshotHandler)
@@ -556,6 +565,21 @@ func apiURLHandler(c *gin.Context) {
 	c.JSON(http.StatusOK, urls)
 }
 
+func apiSearchHandler(c *gin.Context) {
+
+	// use gorm SmartSelect Fields to filter URL
+	search := "%" + c.Query("q") + "%"
+	var urls []storage.URL
+
+	rsDB.
+		Where("URL LIKE ?", search).
+		Or("Title LIKE ?", search).
+		Or("DOM LIKE ?", search).
+		Find(&urls)
+
+	c.JSON(http.StatusOK, urls)
+}
+
 // apiDetailHandler handles a detail request for screenshot information
 func apiDetailHandler(c *gin.Context) {
 

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/rs/zerolog v1.29.1
 	github.com/spf13/cobra v1.7.0
 	github.com/tomsteele/go-nmap v0.0.0-20191202052157-3507e0b03523
+	gorm.io/driver/postgres v1.4.5
 	golang.org/x/net v0.9.0
 	gorm.io/driver/sqlite v1.5.0
 	gorm.io/gorm v1.25.0
@@ -29,6 +30,14 @@ require (
 	github.com/go-playground/validator/v10 v10.13.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.13.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.12.0 // indirect
+	github.com/jackc/pgx/v4 v4.17.2 // indirect
 	github.com/gobwas/ws v1.2.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

storage/db.go

@@ -1,7 +1,9 @@
 package storage
 
 import (
+	"errors"
 	"gorm.io/driver/sqlite"
+	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 )
@@ -14,8 +16,14 @@ type Db struct {
 	// cli flags
 	Disabled bool
 	Debug    bool
+	Platform int
 }
 
+const (
+	Sqlite = iota
+	Postgres
+)
+
 // NewDb sets up a new DB
 func NewDb() *Db {
 	return &Db{}
@@ -35,16 +43,29 @@ func (db *Db) Get() (*gorm.DB, error) {
 		config.Logger = logger.Default.LogMode(logger.Error)
 	}
 
-	conn, err := gorm.Open(sqlite.Open(db.Path+"?cache=shared"), config)
-	if err != nil {
-		return nil, err
-	}
+	switch db.Platform {
+	case Sqlite:
+		conn, err := gorm.Open(sqlite.Open(db.Path+"?cache=shared"), config)
+		if err != nil {
+			return nil, err
+		}
 
-	if !db.SkipMigration {
-		conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}, &Technologie{}, &ConsoleLog{}, &NetworkLog{})
-	}
+		if !db.SkipMigration {
+			conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}, &Technologie{}, &ConsoleLog{}, &NetworkLog{})
+		}
+		return conn, nil
+	case Postgres:
+		conn, err := gorm.Open(postgres.Open(db.Path), config)
+		if err != nil {
+			return nil, err
+		}
 
-	return conn, nil
+		if !db.SkipMigration {
+			conn.AutoMigrate(&URL{}, &Header{}, &TLS{}, &TLSCertificate{}, &TLSCertificateDNSName{}, &Technologie{}, &ConsoleLog{}, &NetworkLog{})
+		}
+		return conn, nil
+	}
+	return nil, errors.New("invalid db platform")
 }
 
 // OrderPerception orders by perception hash if enabled

storage/models.go

@@ -23,6 +23,7 @@ type URL struct {
 	IsPDF          bool
 	PerceptionHash string
 	DOM            string
+	Screenshot     string
 
 	TLS TLS
 

web/ui-templates/detail.html

@@ -61,9 +61,8 @@ <h2 class="page-title">
               <embed class="card-img-top" src='{{ URL "/screenshots/" }}{{ .Data.Filename }}' type="application/pdf" frameBorder="0"
                 scrolling="auto" height="100%" width="100%"></embed>
               {{ else }}
-              <a href='{{ URL "/screenshots/" }}{{ .Data.Filename }}' target="_blank" class="d-block">
-                <img loading="lazy" src='{{ URL "/screenshots/" }}{{ .Data.Filename }}'
-                  onerror="this.onerror=null; this.src='{{ URL "/assets/img/blank.png" }}'" class="card-img-top">
+              <a class="d-block">
+                <img loading="lazy" src="data:image/png;base64,{{ .Data.Screenshot }}"; class="card-img-top">
               </a>
               {{ end }}
               <div class="card-footer">

web/ui-templates/gallery.html

@@ -80,9 +80,8 @@ <h2 class="page-title">
           <embed src='{{ URL "/screenshots/" }}{{ .Filename }}' type="application/pdf" frameBorder="0" scrolling="auto"
             height="100%" width="100%"></embed>
           {{ else }}
-          <a href='{{ URL "/screenshots/" }}{{ .Filename }}' target="_blank" class="d-block">
-            <img loading="lazy" src='{{ URL "/screenshots/" }}{{ .Filename }}'
-              onerror="this.onerror=null; this.src='{{ URL "/assets/img/blank.png" }}'" class="card-img-top">
+          <a class="d-block">
+            <img loading="lazy" src="data:image/png;base64,{{ .Screenshot }}" alt="Screenshot" class="card-img-top">
           </a>
           {{ end }}
           <div class="card-body">