Skip to content

Commit

Permalink
feat(exploitdb): add the information of exploitdb-papers (#52)
Browse files Browse the repository at this point in the history 
* feat(exploitdb): insert the information of exploitdb-papers

* feat(exploitdb): add paper info to search cmd

* feat(models): change JSON output

* fix(fetch): insert ExploitUniqueID in exploitdb

* chore: use regexp.MatchString

* fix(fetch): search by ExploitUniqueID
  • Loading branch information
@MaineK00n
MaineK00n authored Aug 8, 2021
1 parent e5081ca commit 79c6cb4
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 17 deletions.
6 changes: 5 additions & 1 deletion commands/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func searchExploit(cmd *cobra.Command, args []string) (err error) {
var results []*models.Exploit
switch searchType {
case "CVE":
if !cveIDRegexp.Match([]byte(param)) {
if !cveIDRegexp.MatchString(param) {
log15.Error("Specify the search type [CVE] parameters like `--param CVE-xxxx-xxxx`")
return errors.New("Invalid CVE Param")
}
Expand Down Expand Up @@ -97,6 +97,10 @@ func searchExploit(cmd *cobra.Command, args []string) (err error) {
fmt.Println("\n - Exploit Code or Proof of Concept:")
fmt.Printf(" %s\n", os.ShellCode.ShellCodeURL)
}
if os.Paper != nil {
fmt.Println("\n - Paper:")
fmt.Printf(" %s\n", os.Paper.PaperURL)
}
}
fmt.Println("---------------------------------------")
}
Expand Down
18 changes: 15 additions & 3 deletions db/rdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func (r *RDBDriver) MigrateDB() error {
&models.OffensiveSecurity{},
&models.Document{},
&models.ShellCode{},
&models.Paper{},
&models.GitHubRepository{},
).Error; err != nil {
return fmt.Errorf("Failed to migrate. err: %s", err)
Expand All @@ -83,6 +84,7 @@ func (r *RDBDriver) MigrateDB() error {
errs = errs.Add(r.conn.Model(&models.OffensiveSecurity{}).AddIndex("idx_offensive_security_exploit_unique_id", "exploit_unique_id").Error)
errs = errs.Add(r.conn.Model(&models.Document{}).AddIndex("idx_exploit_document_exploit_unique_id", "exploit_unique_id").Error)
errs = errs.Add(r.conn.Model(&models.ShellCode{}).AddIndex("idx_exploit_shell_code_exploit_unique_id", "exploit_unique_id").Error)
errs = errs.Add(r.conn.Model(&models.Paper{}).AddIndex("idx_exploit_paper_exploit_unique_id", "exploit_unique_id").Error)
errs = errs.Add(r.conn.Model(&models.GitHubRepository{}).AddIndex("idx_exploit_github_repository_exploit_unique_id", "exploit_unique_id").Error)

for _, e := range errs {
Expand Down Expand Up @@ -130,6 +132,9 @@ func (r *RDBDriver) deleteAndInsertExploit(conn *gorm.DB, exploitType models.Exp
if err := tx.Where("offensive_security_id = ?", osID.ID).Delete(&models.ShellCode{}).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
if err := tx.Where("offensive_security_id = ?", osID.ID).Delete(&models.Paper{}).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
if err := tx.Delete(&osID).Error; err != nil {
return xerrors.Errorf("Failed to delete: %w", err)
}
Expand Down Expand Up @@ -178,12 +183,12 @@ func (r *RDBDriver) GetExploitByID(exploitUniqueID string) []*models.Exploit {
switch e.ExploitType {
case models.OffensiveSecurityType:
os := &models.OffensiveSecurity{}
errs = errs.Add(r.conn.Preload("Document").Preload("ShellCode").Where(&models.OffensiveSecurity{ExploitUniqueID: e.ExploitUniqueID}).First(&os).Error)
errs = errs.Add(r.conn.Preload("Document").Preload("ShellCode").Preload("Paper").Where(&models.OffensiveSecurity{ExploitID: e.ID}).First(&os).Error)
e.OffensiveSecurity = os

case models.GitHubRepositoryType:
gh := &models.GitHubRepository{}
errs = errs.Add(r.conn.Where(&models.GitHubRepository{ExploitUniqueID: e.ExploitUniqueID}).First(&gh).Error)
errs = errs.Add(r.conn.Where(&models.GitHubRepository{ExploitID: e.ID}).First(&gh).Error)
e.GitHubRepository = gh
}
}
Expand All @@ -200,13 +205,15 @@ func (r *RDBDriver) GetExploitAll() []*models.Exploit {
es := []*models.Exploit{}
docs := []*models.Document{}
shells := []*models.ShellCode{}
papers := []*models.Paper{}
offensiveSecurities := []*models.OffensiveSecurity{}
var errs gorm.Errors

errs = errs.Add(r.conn.Find(&es).Error)
errs = errs.Add(r.conn.Find(&offensiveSecurities).Error)
errs = errs.Add(r.conn.Find(&docs).Error)
errs = errs.Add(r.conn.Find(&shells).Error)
errs = errs.Add(r.conn.Find(&papers).Error)
if len(errs.GetErrors()) > 0 {
log15.Error("Failed to delete old records", "err", errs.Error())
}
Expand All @@ -223,6 +230,11 @@ func (r *RDBDriver) GetExploitAll() []*models.Exploit {
o.ShellCode = s
}
}
for _, p := range papers {
if o.ID == p.OffensiveSecurityID {
o.Paper = p
}
}
if e.ID == o.ExploitID {
e.OffensiveSecurity = o
}
Expand All @@ -249,7 +261,7 @@ func (r *RDBDriver) GetExploitByCveID(cveID string) []*models.Exploit {
switch e.ExploitType {
case models.OffensiveSecurityType:
os := &models.OffensiveSecurity{}
errs = errs.Add(r.conn.Preload("Document").Preload("ShellCode").Where(&models.OffensiveSecurity{ExploitUniqueID: e.ExploitUniqueID}).First(&os).Error)
errs = errs.Add(r.conn.Preload("Document").Preload("ShellCode").Preload("Paper").Where(&models.OffensiveSecurity{ExploitUniqueID: e.ExploitUniqueID}).First(&os).Error)
e.OffensiveSecurity = os

case models.GitHubRepositoryType:
Expand Down
42 changes: 40 additions & 2 deletions fetcher/exploitdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ func FetchExploitDB(deep bool) (exploits []*models.Exploit, err error) {
return nil, err
}

var exploitPaperMap map[string]*models.Paper
if exploitPaperMap, err = FetchExploitPaperMap(); err != nil {
return nil, err
}

// distinct EID(nvd + github)
uniqEIDs := map[string]struct{}{}
for eid := range eidCvesMap {
Expand All @@ -46,12 +51,18 @@ func FetchExploitDB(deep bool) (exploits []*models.Exploit, err error) {
for eid := range exploitDocMap {
uniqEIDs[eid] = struct{}{}
}
for eid := range exploitPaperMap {
uniqEIDs[eid] = struct{}{}
}

for eid := range uniqEIDs {
cveIDs, ok := eidCvesMap[eid]
if ok {
for _, cveID := range cveIDs {
var description string
if e, ok := exploitPaperMap[eid]; ok {
description = e.Description
}
if e, ok := exploitShellCodeMap[eid]; ok {
description = e.Description
}
Expand All @@ -72,13 +83,17 @@ func FetchExploitDB(deep bool) (exploits []*models.Exploit, err error) {
ExploitUniqueID: eid,
Document: exploitDocMap[eid],
ShellCode: exploitShellCodeMap[eid],
Paper: exploitPaperMap[eid],
},
}
exploits = append(exploits, exploit)
}
} else {
// No CveID
var description string
if e, ok := exploitPaperMap[eid]; ok {
description = e.Description
}
if e, ok := exploitShellCodeMap[eid]; ok {
description = e.Description
}
Expand All @@ -94,8 +109,10 @@ func FetchExploitDB(deep bool) (exploits []*models.Exploit, err error) {
URL: "https://www.exploit-db.com/exploits/" + eid,
Description: description,
OffensiveSecurity: &models.OffensiveSecurity{
Document: exploitDocMap[eid],
ShellCode: exploitShellCodeMap[eid],
ExploitUniqueID: eid,
Document: exploitDocMap[eid],
ShellCode: exploitShellCodeMap[eid],
Paper: exploitPaperMap[eid],
},
}
exploits = append(exploits, exploit)
Expand Down Expand Up @@ -286,3 +303,24 @@ func FetchExploitDocumentMap() (eidDocMap map[string]*models.Document, err error
}
return eidDocMap, nil
}

// FetchExploitPaperMap :
func FetchExploitPaperMap() (eidPaperMap map[string]*models.Paper, err error) {
eidPaperMap = map[string]*models.Paper{}
url := "https://raw.githubusercontent.com/offensive-security/exploitdb-papers/master/files_papers.csv"
log15.Info("Fetching", "URL", url)
cveCsv, err := util.FetchURL(url)
if err != nil {
return nil, err
}
papers := []*models.Paper{}
if err := gocsv.UnmarshalBytes(cveCsv, &papers); err != nil {
return nil, err
}

for _, paper := range papers {
paper.PaperURL = "https://github.com/offensive-security/exploitdb-papers/blob/master/" + paper.PaperURL
eidPaperMap[paper.ExploitUniqueID] = paper
}
return eidPaperMap, nil
}
40 changes: 29 additions & 11 deletions models/exploit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ var (

// Exploit :
type Exploit struct {
ID int64 `json:",omitempty"`
ID int64 `json:"-"`
ExploitType ExploitType `json:"exploit_type"`
ExploitUniqueID string `json:"exploit_unique_id"`
URL string `json:"url"`
Expand All @@ -33,9 +33,9 @@ type Exploit struct {

// GitHubRepository :
type GitHubRepository struct {
ID int64 `json:",omitempty"`
ExploitID int64 `sql:"type:bigint REFERENCES exploits(id)" json:",omitempty"`
ExploitUniqueID string `json:"exploit_unique_id"`
ID int64 `json:"-"`
ExploitID int64 `sql:"type:bigint REFERENCES exploits(id)" json:"-"`
ExploitUniqueID string `json:"-"`
Star int `json:"star"`
Fork int `json:"fork"`
CreatedAt time.Time `json:"created_at"`
Expand All @@ -44,18 +44,20 @@ type GitHubRepository struct {

// OffensiveSecurity : https://www.exploit-db.com/
type OffensiveSecurity struct {
ID int64 `json:",omitempty"`
ExploitID int64 `sql:"type:bigint REFERENCES exploits(id)" json:",omitempty"`
ExploitUniqueID string `json:"exploit_unique_id"`
ID int64 `json:"-"`
ExploitID int64 `sql:"type:bigint REFERENCES exploits(id)" json:"-"`
ExploitUniqueID string `json:"-"`
Document *Document `json:"document"`
ShellCode *ShellCode `json:"shell_code"`
Paper *Paper `json:"paper"`
}

// Document :
// https://github.com/offensive-security/exploitdb/tree/master/exploits
type Document struct {
OffensiveSecurityID int64 `sql:"type:bigint REFERENCES offensive_securities(id)" json:",omitempty"`
ExploitUniqueID string `csv:"id" json:"exploit_unique_id"`
// ID int64 `json:"-"`
OffensiveSecurityID int64 `sql:"type:bigint REFERENCES offensive_securities(id)" json:"-"`
ExploitUniqueID string `csv:"id" json:"-"`
DocumentURL string `csv:"file" json:"document_url"`
Description string `csv:"description" json:"description"`
Date OffensiveSecurityTime `csv:"date" json:"date"`
Expand All @@ -69,15 +71,31 @@ type Document struct {
// ShellCode :
// https://github.com/offensive-security/exploitdb/tree/master/shellcodes
type ShellCode struct {
OffensiveSecurityID int64 `sql:"type:bigint REFERENCES offensive_securities(id)" json:",omitempty"`
ExploitUniqueID string `csv:"id" json:"exploit_unique_id"`
// ID int64 `json:"-"`
OffensiveSecurityID int64 `sql:"type:bigint REFERENCES offensive_securities(id)" json:"-"`
ExploitUniqueID string `csv:"id" json:"-"`
ShellCodeURL string `csv:"file" json:"shell_code_url"`
Description string `csv:"description" json:"description"`
Date OffensiveSecurityTime `csv:"date" json:"date"`
Author string `csv:"author" json:"author"`
Platform string `csv:"platform" json:"platform"`
}

// Paper :
// https://github.com/offensive-security/exploitdb-papers/blob/master/files_papers.csv
type Paper struct {
// ID int64 `json:"-"`
OffensiveSecurityID int64 `sql:"type:bigint REFERENCES offensive_securities(id)" json:"-"`
ExploitUniqueID string `csv:"id" json:"-"`
PaperURL string `csv:"file" json:"paper_path"`
Description string `csv:"description" json:"description"`
Date OffensiveSecurityTime `csv:"date" json:"date"`
Author string `csv:"author" json:"author"`
Type string `csv:"type" json:"type"`
Platform string `csv:"platform" json:"platform"`
Language string `csv:"language" json:"language"`
}

// MitreXML :
// http://cve.mitre.org/cve/cvrf.html
type MitreXML struct {
Expand Down

0 comments on commit 79c6cb4

Please sign in to comment.