feat(fetch): delete --deep option (#51)

* feat(github): fill in CreatedAt,UpdatedAt

* chore: change .gitignore

* feat(github): clone PoC dir

* feat(github): reduce noise by limiting num of repository stars and forks

* fix(github): fill in github_repository field in JSON output

* feat(github): can set threshold stars/forks

* feat(github): sort by Star and Fork count when Inserting

* feat(fetch): delete --deep option

* fix(exploitdb): fix the URL to the file on GitHub

* chore: remove comment

README.md

@@ -63,7 +63,6 @@ Global Flags:
       --dbtype string       Database type to store data in (sqlite3, mysql, postgres, or redis supported)
       --debug               debug mode (default: false)
       --debug-sql           SQL debug mode
-      --deep                deep mode extract cve-id from github sources
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON
@@ -77,14 +76,6 @@ Use "go-exploitdb fetch [command] --help" for more information about a command.
 $ go-exploitdb fetch exploitdb
 ```
 
-###### Deep Fetch and Insert Exploit
-- This is very time consuming
-- We will further increase the mapping rate between exploit and cveID.
-- The number of exploits that can be detected remains unchanged
-```bash
-$ go-exploitdb fetch exploitdb -deep
-```
-
 ### Usage: Search Exploits
 ```bash
 $ go-exploitdb search -h
@@ -105,7 +96,6 @@ Global Flags:
       --dbtype string       Database type to store data in (sqlite3, mysql, postgres or redis supported)
       --debug               debug mode (default: false)
       --debug-sql           SQL debug mode
-      --deep                deep mode extract cve-id from github sources
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON
@@ -204,7 +194,6 @@ Global Flags:
       --dbtype string       Database type to store data in (sqlite3, mysql, postgres or redis supported)
       --debug               debug mode (default: false)
       --debug-sql           SQL debug mode
-      --deep                deep mode extract cve-id from github sources
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON

commands/fetch-awesomepoc.go

@@ -35,7 +35,7 @@ func fetchAwesomePoc(cmd *cobra.Command, args []string) (err error) {
 
 	log15.Info("Fetching Awesome Poc Exploit")
 	var exploits []*models.Exploit
-	if exploits, err = fetcher.FetchAwesomePoc(viper.GetBool("deep")); err != nil {
+	if exploits, err = fetcher.FetchAwesomePoc(); err != nil {
 		log15.Error("Failed to fetch AwesomePoc Exploit", "err", err)
 		return err
 	}

commands/fetch-exploitdb.go

@@ -35,7 +35,7 @@ func fetchExploitDB(cmd *cobra.Command, args []string) (err error) {
 
 	log15.Info("Fetching Offensive Security Exploit")
 	var exploits []*models.Exploit
-	if exploits, err = fetcher.FetchExploitDB(viper.GetBool("deep")); err != nil {
+	if exploits, err = fetcher.FetchExploitDB(); err != nil {
 		log15.Error("Failed to fetch Exploit", "err", err)
 		return err
 	}

commands/fetch-githubrepos.go

@@ -41,7 +41,7 @@ func fetchGitHubRepos(cmd *cobra.Command, args []string) (err error) {
 
 	log15.Info("Fetching GitHub Repos Exploit")
 	var exploits []*models.Exploit
-	if exploits, err = fetcher.FetchGitHubRepos(viper.GetBool("deep"), viper.GetInt("threshold-stars"), viper.GetInt("threshold-forks")); err != nil {
+	if exploits, err = fetcher.FetchGitHubRepos(viper.GetInt("threshold-stars"), viper.GetInt("threshold-forks")); err != nil {
 		log15.Error("Failed to fetch GitHubRepo Exploit", "err", err)
 		return err
 	}

commands/root.go

@@ -57,10 +57,6 @@ func init() {
 	_ = viper.BindPFlag("dbtype", RootCmd.PersistentFlags().Lookup("dbtype"))
 	viper.SetDefault("dbtype", "sqlite3")
 
-	RootCmd.PersistentFlags().Bool("deep", false, "deep mode extract cve-id from github sources")
-	_ = viper.BindPFlag("deep", RootCmd.PersistentFlags().Lookup("deep"))
-	viper.SetDefault("deep", false)
-
 	RootCmd.PersistentFlags().String("http-proxy", "", "http://proxy-url:port (default: empty)")
 	_ = viper.BindPFlag("http-proxy", RootCmd.PersistentFlags().Lookup("http-proxy"))
 	viper.SetDefault("http-proxy", "")

config/config.go

@@ -10,7 +10,6 @@ type CommonConfig struct {
 	Debug     bool
 	DebugSQL  bool
 	Quiet     bool
-	Deep      bool
 	DBPath    string
 	DBType    string
 	HTTPProxy string

fetcher/awesomepoc.go

@@ -2,21 +2,95 @@ package fetcher
 
 import (
 	"fmt"
+	"io"
+	"path"
+	"regexp"
+	"strings"
 
 	"github.com/russross/blackfriday/v2"
-	"github.com/vulsio/go-exploitdb/extractor"
 	"github.com/vulsio/go-exploitdb/models"
 	"github.com/vulsio/go-exploitdb/util"
 )
 
+var cveIDRegexpSimple = regexp.MustCompile(`^CVE-\d{4}-\d{4,}$`)
+
+// AwesomePoc :
+type AwesomePoc struct {
+	CveID       string
+	Description string
+	URL         string
+}
+
+// AwesomePocReader :
+type AwesomePocReader struct {
+	Start bool
+	// unique
+	AwesomePoc        map[AwesomePoc]struct{}
+	FillingAwesomePoc AwesomePoc
+}
+
+// RenderNode :
+func (r *AwesomePocReader) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
+	// start extracting from the text "Resource"
+	if string(node.Literal) == "Resource" {
+		r.Start = true
+		r.AwesomePoc = map[AwesomePoc]struct{}{}
+		return 0
+	}
+	if !r.Start {
+		return 0
+	}
+
+	emptyAwesomePoc := AwesomePoc{}
+	switch node.Type {
+	case blackfriday.Text:
+		if cveIDRegexpSimple.Match(node.Literal) {
+			// change awesome poc
+			if r.FillingAwesomePoc != emptyAwesomePoc {
+				r.AwesomePoc[r.FillingAwesomePoc] = struct{}{}
+			}
+			r.FillingAwesomePoc = AwesomePoc{
+				CveID: string(node.Literal),
+			}
+		} else if 0 < len(r.FillingAwesomePoc.CveID) {
+			if 0 < len(node.Literal) {
+				r.FillingAwesomePoc.Description = string(node.Literal)
+			}
+		}
+	case blackfriday.Link:
+		if 0 < len(r.FillingAwesomePoc.CveID) && len(r.FillingAwesomePoc.URL) == 0 {
+			if !strings.Contains(string(node.LinkData.Destination), "http") {
+				// passed relative path of awesome-cve-poc
+				r.FillingAwesomePoc.URL = path.Join("https://github.com/qazbnm456/awesome-cve-poc/blob/master", string(node.LinkData.Destination))
+			} else {
+				r.FillingAwesomePoc.URL = string(node.LinkData.Destination)
+			}
+		}
+	}
+	return 0
+}
+
+// RenderHeader :
+func (r *AwesomePocReader) RenderHeader(w io.Writer, ast *blackfriday.Node) {
+}
+
+// RenderFooter :
+func (r *AwesomePocReader) RenderFooter(w io.Writer, ast *blackfriday.Node) {
+	// 最後のFillingを挿入
+	emptyAwesomePoc := AwesomePoc{}
+	if r.FillingAwesomePoc != emptyAwesomePoc {
+		r.AwesomePoc[r.FillingAwesomePoc] = struct{}{}
+	}
+}
+
 // FetchAwesomePoc :
-func FetchAwesomePoc(deep bool) (exploits []*models.Exploit, err error) {
+func FetchAwesomePoc() (exploits []*models.Exploit, err error) {
 	url := "https://raw.githubusercontent.com/qazbnm456/awesome-cve-poc/master/README.md"
 	readme, err := util.FetchURL(url)
 	if err != nil {
 		return nil, err
 	}
-	r := &extractor.AwesomePocReader{}
+	r := &AwesomePocReader{}
 	blackfriday.Run(readme, blackfriday.WithRenderer(r))
 
 	for poc := range r.AwesomePoc {