From 9226db0ea4537a641b1c19b115e8606c250a2e75 Mon Sep 17 00:00:00 2001 From: MaineK00n Date: Wed, 12 Jun 2024 12:29:27 +0900 Subject: [PATCH] feat(mitre): add new datasource: mitre --- .github/workflows/fetch.yml | 63 +++- README.md | 36 ++ commands/fetchmitre.go | 62 ++++ commands/search.go | 20 +- commands/server.go | 10 +- db/db.go | 2 + db/rdb.go | 211 +++++++++++ db/redis.go | 110 ++++++ fetcher/jvn/jvn.go | 18 +- fetcher/mitre/mitre.go | 675 ++++++++++++++++++++++++++++++++++++ fetcher/mitre/types.go | 327 +++++++++++++++++ models/models.go | 388 +++++++++++++++++++-- models/models_test.go | 57 --- 13 files changed, 1890 insertions(+), 89 deletions(-) create mode 100644 commands/fetchmitre.go create mode 100644 fetcher/mitre/mitre.go create mode 100644 fetcher/mitre/types.go diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml index a5c4782f..65d12071 100644 --- a/.github/workflows/fetch.yml +++ b/.github/workflows/fetch.yml @@ -184,4 +184,65 @@ jobs: run: ./go-cve-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" fortinet - name: fetch redis if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} - run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" fortinet \ No newline at end of file + run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" fortinet + + fetch-mitre: + name: fetch-mitre + runs-on: ubuntu-latest + services: + mysql: + image: mysql + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: test + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + postgres: + image: postgres + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: password + POSTGRES_DB: test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + - name: build + id: build + run: make build + - name: fetch sqlite3 + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./go-cve-dictionary fetch --dbtype sqlite3 mitre + - name: fetch mysql + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./go-cve-dictionary fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" mitre + - name: fetch postgres + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./go-cve-dictionary fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" mitre + - name: fetch redis + if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} + run: ./go-cve-dictionary fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" mitre + diff --git a/README.md b/README.md index d55cca4a..d7172e11 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ Usage: Available Commands: fortinet Fetch Vulnerability dictionary from Fortinet Advisories jvn Fetch Vulnerability dictionary from JVN + mitre Fetch Vulnerability dictionary from MITRE nvd Fetch Vulnerability dictionary from NVD Flags: @@ -292,6 +293,17 @@ $ go-cve-dictionary fetch jvn 2021 $ go-cve-dictionary fetch fortinet ``` +#### Fetch MITRE data +- to fetch all years +```bash +$ go-cve-dictionary fetch mitre +``` + +- to fetch specific years +```bash +$ go-cve-dictionary fetch mitre 2021 +``` + ---- ### Usage: Run HTTP Server @@ -480,6 +492,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal" --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true" ``` +- fetch mitre + + ```bash + $ go-cve-dictionary fetch mitre \ + --dbtype mysql \ + --dbpath "user:pass@tcp(localhost:3306)/dbname?parseTime=true" + ``` + - server ```bash @@ -514,6 +534,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal" --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password" ``` +- fetch mitre + + ```bash + $ go-cve-dictionary fetch mitre \ + --dbtype postgres \ + --dbpath "host=myhost user=user dbname=dbname sslmode=disable password=password" + ``` + - server ```bash @@ -548,6 +576,14 @@ $ go-cve-dictionary search cpe --cveid-only "cpe:/a:fortinet:fortiportal" --dbpath "redis://localhost/0" ``` +- fetch mitre + + ```bash + $ go-cve-dictionary fetch mitre \ + --dbtype redis \ + --dbpath "redis://localhost/0" + ``` + - server ```bash diff --git a/commands/fetchmitre.go b/commands/fetchmitre.go new file mode 100644 index 00000000..1758e85e --- /dev/null +++ b/commands/fetchmitre.go @@ -0,0 +1,62 @@ +package commands + +import ( + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + db "github.com/vulsio/go-cve-dictionary/db" + log "github.com/vulsio/go-cve-dictionary/log" + "github.com/vulsio/go-cve-dictionary/models" + "golang.org/x/xerrors" +) + +var fetchMitreCmd = &cobra.Command{ + Use: "mitre", + Short: "Fetch Vulnerability dictionary from MITRE", + Long: "Fetch Vulnerability dictionary from MITRE", + RunE: fetchMitre, +} + +func init() { + fetchCmd.AddCommand(fetchMitreCmd) +} + +func fetchMitre(_ *cobra.Command, args []string) (err error) { + if err := log.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { + return xerrors.Errorf("Failed to SetLogger. err: %w", err) + } + + driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) + if err != nil { + if xerrors.Is(err, db.ErrDBLocked) { + return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) + } + return xerrors.Errorf("Failed to open DB. err: %w", err) + } + + fetchMeta, err := driver.GetFetchMeta() + if err != nil { + return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) + } + if fetchMeta.OutDated() { + return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) + } + // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + log.Infof("Inserting MITRE into DB (%s).", driver.Name()) + if err := driver.InsertMitre(args); err != nil { + return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + fetchMeta.LastFetchedAt = time.Now() + if err := driver.UpsertFetchMeta(fetchMeta); err != nil { + return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) + } + + log.Infof("Finished fetching MITRE.") + return nil +} diff --git a/commands/search.go b/commands/search.go index ee4ce9d3..b26dadd2 100644 --- a/commands/search.go +++ b/commands/search.go @@ -77,12 +77,20 @@ func searchCVE(_ *cobra.Command, args []string) error { } count += fortinetCount + mitreCount, err := driver.CountMitre() + if err != nil { + log.Errorf("Failed to count MITRE table: %s", err) + return err + } + count += mitreCount + if count == 0 { - log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet") + log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE") log.Infof("") log.Infof(" go-cve-dictionary fetch nvd") log.Infof(" go-cve-dictionary fetch jvn") log.Infof(" go-cve-dictionary fetch fortinet") + log.Infof(" go-cve-dictionary fetch mitre") log.Infof("") return nil } @@ -162,12 +170,20 @@ func searchCPE(_ *cobra.Command, args []string) error { } count += fortinetCount + mitreCount, err := driver.CountMitre() + if err != nil { + log.Errorf("Failed to count MITRE table: %s", err) + return err + } + count += mitreCount + if count == 0 { - log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet") + log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE") log.Infof("") log.Infof(" go-cve-dictionary fetch nvd") log.Infof(" go-cve-dictionary fetch jvn") log.Infof(" go-cve-dictionary fetch fortinet") + log.Infof(" go-cve-dictionary fetch mitre") log.Infof("") return nil } diff --git a/commands/server.go b/commands/server.go index dd4f54e2..1519b8f7 100644 --- a/commands/server.go +++ b/commands/server.go @@ -71,12 +71,20 @@ func executeServer(_ *cobra.Command, _ []string) (err error) { } count += fortinetCount + mitreCount, err := driver.CountMitre() + if err != nil { + log.Errorf("Failed to count MITRE table: %s", err) + return err + } + count += mitreCount + if count == 0 { - log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet") + log.Infof("No Vulnerability data found. Run the below command to fetch data from NVD, JVN, Fortinet, MITRE") log.Infof("") log.Infof(" go-cve-dictionary fetch nvd") log.Infof(" go-cve-dictionary fetch jvn") log.Infof(" go-cve-dictionary fetch fortinet") + log.Infof(" go-cve-dictionary fetch mitre") log.Infof("") return nil } diff --git a/db/db.go b/db/db.go index 09d74c98..d8c9e26a 100644 --- a/db/db.go +++ b/db/db.go @@ -35,9 +35,11 @@ type DB interface { InsertJvn([]string) error InsertNvd([]string) error InsertFortinet([]models.Fortinet) error + InsertMitre([]string) error CountNvd() (int, error) CountJvn() (int, error) CountFortinet() (int, error) + CountMitre() (int, error) } // Option : diff --git a/db/rdb.go b/db/rdb.go index 43e68ab9..357504a8 100644 --- a/db/rdb.go +++ b/db/rdb.go @@ -23,6 +23,7 @@ import ( "github.com/vulsio/go-cve-dictionary/config" "github.com/vulsio/go-cve-dictionary/fetcher/jvn" + "github.com/vulsio/go-cve-dictionary/fetcher/mitre" "github.com/vulsio/go-cve-dictionary/fetcher/nvd" logger "github.com/vulsio/go-cve-dictionary/log" "github.com/vulsio/go-cve-dictionary/models" @@ -167,6 +168,43 @@ func (r *RDBDriver) MigrateDB() error { &models.FortinetCwe{}, &models.FortinetCpe{}, &models.FortinetReference{}, + + &models.Mitre{}, + &models.MitreCVEMetadata{}, + &models.MitreContainer{}, + &models.MitreProviderMetadata{}, + &models.MitreDescription{}, + &models.MitreDescriptionSupportingMedia{}, + // &models.MitreCpe{}, + &models.MitreProblemType{}, + &models.MitreProblemTypeDescription{}, + &models.MitreProblemTypeDescriptionReference{}, + &models.MitreImpact{}, + &models.MitreImpactDescription{}, + &models.MitreImpactDescriptionSupportingMedia{}, + &models.MitreMetric{}, + &models.MitreMetricScenario{}, + &models.MitreMetricCVSS2{}, + &models.MitreMetricCVSS30{}, + &models.MitreMetricCVSS31{}, + &models.MitreMetricCVSS40{}, + &models.MitreMetricSSVC{}, + &models.MitreMetricKEV{}, + &models.MitreMetricOther{}, + &models.MitreWorkaround{}, + &models.MitreWorkaroundSupportingMedia{}, + &models.MitreSolution{}, + &models.MitreSolutionSupportingMedia{}, + &models.MitreExploit{}, + &models.MitreExploitSupportingMedia{}, + &models.MitreConfiguration{}, + &models.MitreConfigurationSupportingMedia{}, + &models.MitreReference{}, + &models.MitreTimeline{}, + &models.MitreCredit{}, + &models.MitreTag{}, + &models.MitreTaxonomyMapping{}, + &models.MitreTaxonomyRelation{}, ); err != nil { switch r.name { case dialectSqlite3: @@ -304,6 +342,45 @@ func (r *RDBDriver) Get(cveID string) (*models.CveDetail, error) { return nil, xerrors.Errorf("Failed to fill Fortinet. Fortinet{CveID: %s} err: %w", cveID, err) } + if err := r.conn. + InnerJoins("CVEMetadata", r.conn.Where(&models.MitreCVEMetadata{CVEID: cveID})). + Preload("Containers"). + Preload("Containers.ProviderMetadata"). + Preload("Containers.Descriptions"). + Preload("Containers.Descriptions.SupportingMedia"). + Preload("Containers.ProblemTypes"). + Preload("Containers.ProblemTypes.Descriptions"). + Preload("Containers.ProblemTypes.Descriptions.References"). + Preload("Containers.Impacts"). + Preload("Containers.Impacts.Descriptions"). + Preload("Containers.Impacts.Descriptions.SupportingMedia"). + Preload("Containers.Metrics"). + Preload("Containers.Metrics.Scenarios"). + Preload("Containers.Metrics.CVSSv2"). + Preload("Containers.Metrics.CVSSv30"). + Preload("Containers.Metrics.CVSSv31"). + Preload("Containers.Metrics.CVSSv40"). + Preload("Containers.Metrics.SSVC"). + Preload("Containers.Metrics.KEV"). + Preload("Containers.Metrics.Other"). + Preload("Containers.Workarounds"). + Preload("Containers.Workarounds.SupportingMedia"). + Preload("Containers.Solutions"). + Preload("Containers.Solutions.SupportingMedia"). + Preload("Containers.Exploits"). + Preload("Containers.Exploits.SupportingMedia"). + Preload("Containers.Configurations"). + Preload("Containers.Configurations.SupportingMedia"). + Preload("Containers.References"). + Preload("Containers.Timeline"). + Preload("Containers.Credits"). + Preload("Containers.Tags"). + Preload("Containers.TaxonomyMappings"). + Preload("Containers.TaxonomyMappings.TaxonomyRelations"). + Find(&detail.Mitres).Error; err != nil { + return nil, xerrors.Errorf("Failed to fill Mitre. Mitre{CveID: %s} err: %w", cveID, err) + } + return &detail, nil } @@ -335,6 +412,14 @@ func (r *RDBDriver) GetCveIDs() ([]string, error) { cveIDs[cveID] = struct{}{} } + var mitres []string + if err := r.conn.Model(&models.MitreCVEMetadata{}).Distinct().Pluck("cve_id", &mitres).Error; err != nil { + return nil, err + } + for _, cveID := range mitres { + cveIDs[cveID] = struct{}{} + } + return maps.Keys(cveIDs), nil } @@ -732,3 +817,129 @@ func deleteFortinet(tx *gorm.DB) error { } return nil } + +// CountMitre count mitre table +func (r *RDBDriver) CountMitre() (int, error) { + var count int64 + if err := r.conn.Model(&models.Mitre{}).Count(&count).Error; err != nil { + return 0, err + } + return int(count), nil +} + +// InsertMitre Cve information from DB. +func (r *RDBDriver) InsertMitre(years []string) (err error) { + tx := r.conn.Begin() + defer func() { + if re := recover(); re != nil { + tx.Rollback() + } + }() + if err := tx.Error; err != nil { + return err + } + + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + + logger.Infof("Deleting MITRE tables...") + if err := deleteMitre(tx); err != nil { + tx.Rollback() + return xerrors.Errorf("Failed to deleteMitre. err: %w", err) + } + + logger.Infof("Fetching CVE information from MITRE.") + cacheDir, err := mitre.Fetch() + if err != nil { + tx.Rollback() + return xerrors.Errorf("Failed to Fetch. err: %w", err) + } + defer os.RemoveAll(cacheDir) + + if len(years) == 0 { + for y := 1999; y <= time.Now().Year(); y++ { + years = append(years, fmt.Sprintf("%d", y)) + } + } + for _, year := range years { + logger.Infof("Fetching CVE information from MITRE(%s).", year) + cves, err := mitre.Convert(cacheDir, year) + if err != nil { + tx.Rollback() + return xerrors.Errorf("Failed to Convert. err: %w", err) + } + + logger.Infof("Inserting fetched CVEs(%s)...", year) + if err := insertMitre(tx, cves, batchSize); err != nil { + tx.Rollback() + return xerrors.Errorf("Failed to insertMitre. err: %w", err) + } + logger.Infof("Refreshed %d CVEs.", len(cves)) + } + + if err := tx.Commit().Error; err != nil { + return xerrors.Errorf("Failed to Commit Transaction. err: %w", err) + } + return nil +} + +func deleteMitre(tx *gorm.DB) error { + for _, table := range []interface{}{ + models.Mitre{}, + models.MitreCVEMetadata{}, + models.MitreContainer{}, + models.MitreProviderMetadata{}, + models.MitreDescription{}, + models.MitreDescriptionSupportingMedia{}, + // models.MitreCpe{}, + models.MitreProblemType{}, + models.MitreProblemTypeDescription{}, + models.MitreProblemTypeDescriptionReference{}, + models.MitreImpact{}, + models.MitreImpactDescription{}, + models.MitreImpactDescriptionSupportingMedia{}, + models.MitreMetric{}, + models.MitreMetricScenario{}, + models.MitreMetricCVSS2{}, + models.MitreMetricCVSS30{}, + models.MitreMetricCVSS31{}, + models.MitreMetricCVSS40{}, + models.MitreMetricSSVC{}, + models.MitreMetricKEV{}, + models.MitreMetricOther{}, + models.MitreWorkaround{}, + models.MitreWorkaroundSupportingMedia{}, + models.MitreSolution{}, + models.MitreSolutionSupportingMedia{}, + models.MitreExploit{}, + models.MitreExploitSupportingMedia{}, + models.MitreConfiguration{}, + models.MitreConfigurationSupportingMedia{}, + models.MitreReference{}, + models.MitreTimeline{}, + models.MitreCredit{}, + models.MitreTag{}, + models.MitreTaxonomyMapping{}, + models.MitreTaxonomyRelation{}, + } { + if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(table).Error; err != nil { + return xerrors.Errorf("Failed to delete old records. err: %w", err) + } + } + return nil +} + +func insertMitre(tx *gorm.DB, cves []models.Mitre, _ int) error { + bar := pb.StartNew(len(cves)) + for _, cve := range cves { + if err := tx.Create(&cve).Error; err != nil { + return xerrors.Errorf("Failed to insert. err: %w", err) + } + bar.Increment() + } + bar.Finish() + + return nil +} diff --git a/db/redis.go b/db/redis.go index 28ab97bf..98aae664 100644 --- a/db/redis.go +++ b/db/redis.go @@ -19,6 +19,7 @@ import ( "github.com/vulsio/go-cve-dictionary/config" "github.com/vulsio/go-cve-dictionary/fetcher/jvn" + "github.com/vulsio/go-cve-dictionary/fetcher/mitre" "github.com/vulsio/go-cve-dictionary/fetcher/nvd" log "github.com/vulsio/go-cve-dictionary/log" "github.com/vulsio/go-cve-dictionary/models" @@ -197,6 +198,7 @@ func convertToCveDetail(cveID string, results map[string]string) (models.CveDeta Nvds: []models.Nvd{}, Jvns: []models.Jvn{}, Fortinets: []models.Fortinet{}, + Mitres: []models.Mitre{}, } for field, jsonStr := range results { @@ -219,6 +221,12 @@ func convertToCveDetail(cveID string, results map[string]string) (models.CveDeta return models.CveDetail{}, xerrors.Errorf("Failed to Unmarshal JSON. err: %w", err) } detail.Fortinets = append(detail.Fortinets, fortinet) + case field == models.MitreType: + var mitre models.Mitre + if err := json.Unmarshal([]byte(jsonStr), &mitre); err != nil { + return models.CveDetail{}, xerrors.Errorf("Failed to Unmarshal JSON. err: %w", err) + } + detail.Mitres = append(detail.Mitres, mitre) default: log.Warnf("field(%s) obtained by %s is not in NVD, JVN, Fortinet format", field, cveID) } @@ -739,3 +747,105 @@ func (r *RedisDriver) InsertFortinet(advs []models.Fortinet) error { return nil } + +// CountMitre count mitre table +func (r *RedisDriver) CountMitre() (int, error) { + depstr, err := r.conn.HGet(context.Background(), depKey, models.MitreType).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return 0, nil + } + return 0, err + } + + // deps: {"CVEID": {}} + var deps map[string]struct{} + if err := json.Unmarshal([]byte(depstr), &deps); err != nil { + return 0, xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) + } + + return len(deps), nil +} + +// InsertMitre Cve information from DB. +func (r *RedisDriver) InsertMitre(years []string) error { + ctx := context.Background() + batchSize := viper.GetInt("batch-size") + if batchSize < 1 { + return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") + } + var err error + + log.Infof("Fetching CVE information from MITRE.") + cacheDir, err := mitre.Fetch() + if err != nil { + return xerrors.Errorf("Failed to Fetch. err: %w", err) + } + defer os.RemoveAll(cacheDir) + + // newDeps, oldDeps: {"CVEID": {}} + newDeps := map[string]struct{}{} + oldDepsStr, err := r.conn.HGet(ctx, depKey, models.MitreType).Result() + if err != nil { + if !errors.Is(err, redis.Nil) { + return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) + } + oldDepsStr = "{}" + } + var oldDeps map[string]map[string]struct{} + if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { + return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) + } + + if len(years) == 0 { + for y := 1999; y <= time.Now().Year(); y++ { + years = append(years, fmt.Sprintf("%d", y)) + } + } + for _, year := range years { + log.Infof("Fetching CVE information from MITRE(%s).", year) + cves, err := mitre.Convert(cacheDir, year) + if err != nil { + return xerrors.Errorf("Failed to Convert. err: %w", err) + } + + log.Infof("Inserting fetched CVEs(%s)...", year) + bar := pb.StartNew(len(cves)) + for idx := range chunkSlice(len(cves), batchSize) { + pipe := r.conn.Pipeline() + for _, cve := range cves[idx.From:idx.To] { + bs, err := json.Marshal(cve) + if err != nil { + return xerrors.Errorf("Failed to marshal json. err: %w", err) + } + + _ = pipe.HSet(ctx, fmt.Sprintf(cveKeyFormat, cve.CVEMetadata.CVEID), models.MitreType, string(bs)) + newDeps[cve.CVEMetadata.CVEID] = struct{}{} + delete(oldDeps, cve.CVEMetadata.CVEID) + } + if _, err = pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + bar.Add(idx.To - idx.From) + } + bar.Finish() + log.Infof("Refreshed %d CVEs.", len(cves)) + } + + pipe := r.conn.Pipeline() + for cveID := range oldDeps { + if _, ok := newDeps[cveID]; !ok { + _ = pipe.HDel(ctx, fmt.Sprintf(cveKeyFormat, cveID), models.MitreType) + } + } + newDepsJSON, err := json.Marshal(newDeps) + if err != nil { + return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) + } + _ = pipe.HSet(ctx, depKey, models.MitreType, string(newDepsJSON)) + if _, err = pipe.Exec(ctx); err != nil { + return xerrors.Errorf("Failed to exec pipeline. err: %w", err) + } + + return nil +} diff --git a/fetcher/jvn/jvn.go b/fetcher/jvn/jvn.go index d2511f68..245bf6b1 100644 --- a/fetcher/jvn/jvn.go +++ b/fetcher/jvn/jvn.go @@ -86,12 +86,18 @@ func FetchConvert(uniqCve map[string]map[string]models.Jvn, years []string) erro } func fetch(year string) ([]Item, error) { - url, err := models.GetURLByYear(models.JvnType, year) - if err != nil { - return nil, xerrors.Errorf("Failed to GetURLByYear. err: %w", err) - } + u := func() string { + switch year { + case "modified": + return "https://jvndb.jvn.jp/ja/rss/jvndb.rdf" + case "recent": + return "https://jvndb.jvn.jp/ja/rss/jvndb_new.rdf" + default: + return fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/years/jvndb_%s.rdf", year) + } + }() - body, err := fetcher.FetchFeedFile(url, false) + body, err := fetcher.FetchFeedFile(u, false) if err != nil { return nil, xerrors.Errorf("Failed to FetchFeedFiles. err: %w", err) } @@ -99,7 +105,7 @@ func fetch(year string) ([]Item, error) { var rdf rdf items := []Item{} if err := xml.Unmarshal([]byte(body), &rdf); err != nil { - return nil, xerrors.Errorf("Failed to unmarshal. url: %s, err: %w", url, err) + return nil, xerrors.Errorf("Failed to unmarshal. url: %s, err: %w", u, err) } for i, item := range rdf.Items { if !(strings.Contains(item.Description, "** 未確定 **") || strings.Contains(item.Description, "** サポート外 **") || strings.Contains(item.Description, "** 削除 **")) { diff --git a/fetcher/mitre/mitre.go b/fetcher/mitre/mitre.go new file mode 100644 index 00000000..7a7aaccf --- /dev/null +++ b/fetcher/mitre/mitre.go @@ -0,0 +1,675 @@ +package mitre + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "golang.org/x/xerrors" + + "github.com/vulsio/go-cve-dictionary/fetcher" + "github.com/vulsio/go-cve-dictionary/log" + "github.com/vulsio/go-cve-dictionary/models" + "github.com/vulsio/go-cve-dictionary/util" +) + +const repositoryURL = "https://github.com/CVEProject/cvelistV5/archive/refs/heads/main.tar.gz" + +func Fetch() (string, error) { + d := util.CacheDir() + + if err := os.RemoveAll(d); err != nil { + return "", xerrors.Errorf("Failed to remove %s. err: %w", d, err) + } + + bs, err := fetcher.FetchFeedFile(repositoryURL, false) + if err != nil { + return "", xerrors.Errorf("Failed to fetch. err: %w", err) + } + + gr, err := gzip.NewReader(bytes.NewReader(bs)) + if err != nil { + return "", xerrors.Errorf("Failed to create gzip reader. err: %w", err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", xerrors.Errorf("Failed to next tar reader. err: %w", err) + } + + if hdr.FileInfo().IsDir() { + continue + } + + if !strings.HasPrefix(filepath.Base(hdr.Name), "CVE-") { + continue + } + + if err := func() error { + ss := strings.Split(filepath.Base(hdr.Name), "-") + if len(ss) != 3 { + return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(hdr.Name)) + } + if _, err := time.Parse("2006", ss[1]); err != nil { + return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(hdr.Name)) + } + + if err := os.MkdirAll(filepath.Join(d, ss[1]), os.ModePerm); err != nil { + return xerrors.Errorf("Failed to mkdir %s. err: %w", filepath.Join(d, ss[1]), err) + } + + f, err := os.Create(filepath.Join(d, ss[1], filepath.Base(hdr.Name))) + if err != nil { + return xerrors.Errorf("Failed to create %s. err: %w", filepath.Join(d, ss[1], filepath.Base(hdr.Name)), err) + } + defer f.Close() + + if _, err := io.Copy(f, tr); err != nil { + return xerrors.Errorf("Failed to copy %s to %s. err: %w", hdr.Name, filepath.Join(d, ss[1], filepath.Base(hdr.Name)), err) + } + + return nil + }(); err != nil { + return "", xerrors.Errorf("Failed to extract %s. err: %w", hdr.Name, err) + } + } + + return d, nil +} + +// Convert convert //CVE--\d{4,}.json to []models.Mitre +func Convert(cacheDir, year string) ([]models.Mitre, error) { + var ps []string + if err := filepath.WalkDir(cacheDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + if strings.HasPrefix(filepath.Base(path), fmt.Sprintf("CVE-%s-", year)) { + ps = append(ps, path) + } + + return nil + }); err != nil { + return nil, xerrors.Errorf("Failed to walk %s. err: %w", cacheDir, err) + } + + reqChan := make(chan string, len(ps)) + resChan := make(chan *models.Mitre, len(ps)) + errChan := make(chan error, len(ps)) + defer close(reqChan) + defer close(resChan) + defer close(errChan) + + go func() { + for _, p := range ps { + reqChan <- p + } + }() + + concurrency := runtime.NumCPU() + 2 + tasks := util.GenWorkers(concurrency) + for range ps { + tasks <- func() { + req := <-reqChan + cve, err := convertToModel(req) + if err != nil { + errChan <- err + return + } + resChan <- cve + } + } + + cves := []models.Mitre{} + errs := []error{} + timeout := time.After(10 * 60 * time.Second) + for range ps { + select { + case res := <-resChan: + if res != nil { + cves = append(cves, *res) + } + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + return nil, fmt.Errorf("Timeout Fetching") + } + } + if 0 < len(errs) { + return nil, xerrors.Errorf("%w", errs) + } + return cves, nil +} + +// convertToModel converts Mitre JSON to model structure. +func convertToModel(cvePath string) (*models.Mitre, error) { + f, err := os.Open(cvePath) + if err != nil { + return nil, xerrors.Errorf("Failed to open %s. err: %w", cvePath, err) + } + defer f.Close() + + var item cve + if err := json.NewDecoder(f).Decode(&item); err != nil { + return nil, xerrors.Errorf("Failed to decode %s. err: %w", cvePath, err) + } + + if item.CVEMetadata.State == "REJECTED" { + return nil, nil + } + + return &models.Mitre{ + CVEMetadata: models.MitreCVEMetadata{ + CVEID: item.CVEMetadata.CVEID, + AssignerOrgID: item.CVEMetadata.AssignerOrgID, + AssignerShortName: item.CVEMetadata.AssignerShortName, + RequesterUserID: item.CVEMetadata.RequesterUserID, + Serial: item.CVEMetadata.Serial, + State: item.CVEMetadata.State, + DatePublished: parseTime(layouts, item.CVEMetadata.DatePublished), + DateUpdated: parseTime(layouts, item.CVEMetadata.DateUpdated), + DateReserved: parseTime(layouts, item.CVEMetadata.DateReserved), + }, + Containers: convertContainers(item.Containers.CNA, item.Containers.ADP), + }, nil +} + +var layouts = []string{"2006-01-02", "2006-01-02T15:04:05", "2006-01-02T15:04:05Z", "2006-01-02T15:04:05.000Z", "2006-01-02T15:04:05.000-07:00", "2006-01-02T15:04:05.000000", "2006-01-02T15:04:05.000000Z", "2006-01-02T15:04:05-07:00"} + +func parseTime(layouts []string, value *string) *time.Time { + if value == nil || *value == "" { + return nil + } + for _, l := range layouts { + t, err := time.Parse(l, *value) + if err == nil { + return &t + } + } + log.Warnf("no layout to adapt to %q", *value) + return nil +} + +func convertContainers(cna cna, adps []adp) []models.MitreContainer { + cs := make([]models.MitreContainer, 0, 1+len(adps)) + cs = append(cs, models.MitreContainer{ + ContainerType: "CNA", + ProviderMetadata: models.MitreProviderMetadata{ + OrgID: cna.ProviderMetadata.OrgID, + ShortName: cna.ProviderMetadata.ShortName, + DateUpdated: parseTime(layouts, cna.ProviderMetadata.DateUpdated), + }, + Title: cna.Title, + Descriptions: convertDescription(cna.Descriptions), + ProblemTypes: convertProblemType(cna.ProblemTypes), + Impacts: convertImpact(cna.Impacts), + Metrics: convertMetric(cna.Metrics), + Workarounds: convertWorkaround(cna.Workarounds), + Solutions: convertSolution(cna.Solutions), + Exploits: convertExploit(cna.Exploits), + Configurations: convertConfiguration(cna.Configurations), + References: covnertReference(cna.References), + Timeline: convertTimeline(cna.Timeline), + Credits: convertCredit(cna.Credits), + Source: string(cna.Source), + Tags: convertTag(cna.Tags), + TaxonomyMappings: convertTaxonomyMappings(cna.TaxonomyMappings), + DateAssigned: parseTime(layouts, cna.DateAssigned), + DatePublic: parseTime(layouts, cna.DatePublic), + }) + for _, adp := range adps { + cs = append(cs, models.MitreContainer{ + ContainerType: "ADP", + ProviderMetadata: models.MitreProviderMetadata{ + OrgID: adp.ProviderMetadata.OrgID, + ShortName: adp.ProviderMetadata.ShortName, + DateUpdated: parseTime(layouts, adp.ProviderMetadata.DateUpdated), + }, + Title: adp.Title, + Descriptions: convertDescription(adp.Descriptions), + ProblemTypes: convertProblemType(adp.ProblemTypes), + Impacts: convertImpact(adp.Impacts), + Metrics: convertMetric(adp.Metrics), + Workarounds: convertWorkaround(adp.Workarounds), + Solutions: convertSolution(adp.Solutions), + Exploits: convertExploit(adp.Exploits), + Configurations: convertConfiguration(adp.Configurations), + References: covnertReference(adp.References), + Timeline: convertTimeline(adp.Timeline), + Credits: convertCredit(adp.Credits), + Source: string(adp.Source), + Tags: convertTag(adp.Tags), + TaxonomyMappings: convertTaxonomyMappings(adp.TaxonomyMappings), + DatePublic: parseTime(layouts, adp.DatePublic), + }) + } + return cs +} + +func convertDescription(descriptions []description) []models.MitreDescription { + ds := make([]models.MitreDescription, 0, len(descriptions)) + for _, d := range descriptions { + ms := make([]models.MitreDescriptionSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreDescriptionSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreDescription{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + return ds +} + +func convertProblemType(problemTypes []problemType) []models.MitreProblemType { + ps := make([]models.MitreProblemType, 0, len(problemTypes)) + for _, p := range problemTypes { + ds := make([]models.MitreProblemTypeDescription, 0, len(p.Descriptions)) + for _, d := range p.Descriptions { + rs := make([]models.MitreProblemTypeDescriptionReference, 0, len(d.References)) + for _, r := range d.References { + rs = append(rs, models.MitreProblemTypeDescriptionReference{ + Reference: models.Reference{ + Link: r.URL, + Tags: strings.Join(r.Tags, ","), + Name: func() string { + if r.Name == nil { + return "" + } + return *r.Name + }(), + }, + }) + } + ds = append(ds, models.MitreProblemTypeDescription{ + Type: d.Type, + Lang: d.Lang, + Description: d.Description, + CweID: d.CweID, + References: rs, + }) + } + ps = append(ps, models.MitreProblemType{Descriptions: ds}) + } + return ps +} + +func convertImpact(impacts []impact) []models.MitreImpact { + is := make([]models.MitreImpact, 0, len(impacts)) + for _, i := range impacts { + ds := make([]models.MitreImpactDescription, 0, len(i.Descriptions)) + for _, d := range i.Descriptions { + ms := make([]models.MitreImpactDescriptionSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreImpactDescriptionSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreImpactDescription{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + is = append(is, models.MitreImpact{ + Descriptions: ds, + CapecID: i.CapecID, + }) + } + return is +} + +func convertMetric(metrics []metric) []models.MitreMetric { + var ms []models.MitreMetric + for _, m := range metrics { + sc := make([]models.MitreMetricScenario, 0, len(m.Scenarios)) + for _, s := range m.Scenarios { + sc = append(sc, models.MitreMetricScenario{ + Lang: s.Lang, + Value: s.Value, + }) + } + + if m.CVSSv2 != nil { + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "CVSS" + }(), + Scenarios: sc, + CVSSv2: &models.MitreMetricCVSS2{ + Cvss2: models.Cvss2{ + VectorString: m.CVSSv2.VectorString, + BaseScore: m.CVSSv2.BaseScore, + }, + }, + }) + } + if m.CVSSv30 != nil { + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "CVSS" + }(), + Scenarios: sc, + CVSSv30: &models.MitreMetricCVSS30{ + Cvss3: models.Cvss3{ + VectorString: m.CVSSv30.VectorString, + BaseScore: m.CVSSv30.BaseScore, + BaseSeverity: m.CVSSv30.BaseSeverity, + }, + }, + }) + } + if m.CVSSv31 != nil { + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "CVSS" + }(), + Scenarios: sc, + CVSSv31: &models.MitreMetricCVSS31{ + Cvss3: models.Cvss3{ + VectorString: m.CVSSv31.VectorString, + BaseScore: m.CVSSv31.BaseScore, + BaseSeverity: m.CVSSv31.BaseSeverity, + }, + }, + }) + } + if m.CVSSv40 != nil { + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "CVSS" + }(), + Scenarios: sc, + CVSSv40: &models.MitreMetricCVSS40{ + Cvss40: models.Cvss40{ + VectorString: m.CVSSv40.VectorString, + BaseScore: m.CVSSv40.BaseScore, + BaseSeverity: m.CVSSv40.BaseSeverity, + ThreatScore: m.CVSSv40.ThreatScore, + ThreatSeverity: m.CVSSv40.ThreatSeverity, + EnvironmentalScore: m.CVSSv40.EnvironmentalScore, + EnvironmentalSeverity: m.CVSSv40.EnvironmentalSeverity, + }, + }, + }) + } + if m.Other != nil { + switch m.Other.Type { + case "kev": + var v kev + if err := json.Unmarshal(m.Other.Content, &v); err != nil { + log.Warnf("%v is not expected kev type", m.Other.Content) + break + } + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "KEV" + }(), + Scenarios: sc, + KEV: &models.MitreMetricKEV{ + DateAdded: func() time.Time { + if t := parseTime(layouts, &v.DateAdded); t != nil { + return *t + } + return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) + }(), + Reference: v.Reference, + }, + }) + case "ssvc": + var v ssvc + if err := json.Unmarshal(m.Other.Content, &v); err != nil { + log.Warnf("%v is not expected ssvc type", m.Other.Content) + break + } + s := models.MitreMetricSSVC{ + Role: v.Role, + Version: v.Version, + Timestamp: func() time.Time { + if t := parseTime(layouts, &v.Timestamp); t != nil { + return *t + } + return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) + }(), + } + for _, o := range v.Options { + if o.Exploitation != nil { + s.Exploitation = o.Exploitation + } + if o.Automatable != nil { + s.Automatable = o.Automatable + } + if o.TechnicalImpact != nil { + s.TechnicalImpact = o.TechnicalImpact + } + } + + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "SSVC" + }(), + Scenarios: sc, + SSVC: &s, + }) + default: + ms = append(ms, models.MitreMetric{ + Format: func() string { + if m.Format != nil { + return *m.Format + } + return "other" + }(), + Scenarios: sc, + Other: &models.MitreMetricOther{ + Type: m.Other.Type, + Content: string(m.Other.Content), + }, + }) + } + } + } + return ms +} + +func convertWorkaround(descriptions []description) []models.MitreWorkaround { + ds := make([]models.MitreWorkaround, 0, len(descriptions)) + for _, d := range descriptions { + ms := make([]models.MitreWorkaroundSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreWorkaroundSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreWorkaround{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + return ds +} + +func convertSolution(descriptions []description) []models.MitreSolution { + ds := make([]models.MitreSolution, 0, len(descriptions)) + for _, d := range descriptions { + ms := make([]models.MitreSolutionSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreSolutionSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreSolution{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + return ds +} + +func convertExploit(descriptions []description) []models.MitreExploit { + ds := make([]models.MitreExploit, 0, len(descriptions)) + for _, d := range descriptions { + ms := make([]models.MitreExploitSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreExploitSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreExploit{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + return ds +} + +func convertConfiguration(descriptions []description) []models.MitreConfiguration { + ds := make([]models.MitreConfiguration, 0, len(descriptions)) + for _, d := range descriptions { + ms := make([]models.MitreConfigurationSupportingMedia, 0, len(d.SupportingMedia)) + for _, m := range d.SupportingMedia { + ms = append(ms, models.MitreConfigurationSupportingMedia{ + Type: m.Type, + Base64: m.Base64, + Value: m.Value, + }) + } + ds = append(ds, models.MitreConfiguration{ + Lang: d.Lang, + Value: d.Value, + SupportingMedia: ms, + }) + } + return ds +} + +func covnertReference(references []reference) []models.MitreReference { + rs := make([]models.MitreReference, 0, len(references)) + for _, r := range references { + rs = append(rs, models.MitreReference{ + Reference: models.Reference{ + Link: r.URL, + Tags: strings.Join(r.Tags, ","), + Name: func() string { + if r.Name == nil { + return "" + } + return *r.Name + }(), + }, + }) + } + return rs +} + +func convertTimeline(timeline timeline) []models.MitreTimeline { + ts := make([]models.MitreTimeline, 0, len(timeline)) + for _, t := range timeline { + ts = append(ts, models.MitreTimeline{ + Time: func() time.Time { + if t := parseTime(layouts, &t.Time); t != nil { + return *t + } + return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) + }(), + Lang: t.Lang, + Value: t.Value, + }) + } + return ts +} + +func convertCredit(credits credits) []models.MitreCredit { + cs := make([]models.MitreCredit, 0, len(credits)) + for _, c := range credits { + cs = append(cs, models.MitreCredit{ + Type: c.Type, + Lang: c.Lang, + User: c.User, + Value: c.Value, + }) + } + return cs +} + +func convertTag(tags []string) []models.MitreTag { + ts := make([]models.MitreTag, 0, len(tags)) + for _, t := range tags { + ts = append(ts, models.MitreTag{ + Tag: t, + }) + } + return ts +} + +func convertTaxonomyMappings(taxonomyMappings taxonomyMappings) []models.MitreTaxonomyMapping { + ts := make([]models.MitreTaxonomyMapping, 0, len(taxonomyMappings)) + for _, t := range taxonomyMappings { + rs := make([]models.MitreTaxonomyRelation, 0, len(t.TaxonomyRelations)) + for _, r := range t.TaxonomyRelations { + rs = append(rs, models.MitreTaxonomyRelation{ + TaxonomyID: r.TaxonomyID, + RelationshipName: r.RelationshipName, + RelationshipValue: r.RelationshipValue, + }) + } + ts = append(ts, models.MitreTaxonomyMapping{ + TaxonomyVersion: t.TaxonomyVersion, + TaxonomyName: t.TaxonomyName, + TaxonomyRelations: rs, + }) + } + return ts +} diff --git a/fetcher/mitre/types.go b/fetcher/mitre/types.go new file mode 100644 index 00000000..c6e9f2cc --- /dev/null +++ b/fetcher/mitre/types.go @@ -0,0 +1,327 @@ +package mitre + +import "encoding/json" + +// https://github.com/MaineK00n/vuls-data-update/blob/6c082eefa485f54c210d4cd636e295408cc422bf/pkg/fetch/mitre/v5/types.go#L3-L303 +type cve struct { + DataType string `json:"dataType"` + DataVersion string `json:"dataVersion"` + CVEMetadata cveMetadata `json:"cveMetadata"` + Containers struct { + CNA cna `json:"cna"` + ADP []adp `json:"adp,omitempty"` + } `json:"containers"` +} + +type cveMetadata struct { + CVEID string `json:"cveId"` + AssignerOrgID string `json:"assignerOrgId"` + AssignerShortName *string `json:"assignerShortName,omitempty"` + RequesterUserID *string `json:"requesterUserId,omitempty"` + Serial *int `json:"serial,omitempty"` + State string `json:"state"` + DatePublished *string `json:"datePublished,omitempty"` + DateUpdated *string `json:"dateUpdated,omitempty"` + DateReserved *string `json:"dateReserved,omitempty"` + DateRejected *string `json:"dateRejected,omitempty"` +} + +type cna struct { + ProviderMetadata providerMetadata `json:"providerMetadata"` + Title *string `json:"title,omitempty"` + Descriptions []description `json:"descriptions,omitempty"` + Affected []product `json:"affected,omitempty"` + ProblemTypes []problemType `json:"problemTypes,omitempty"` + Impacts []impact `json:"impacts,omitempty"` + Metrics []metric `json:"metrics,omitempty"` + Workarounds []description `json:"workarounds,omitempty"` + Solutions []description `json:"solutions,omitempty"` + Exploits []description `json:"exploits,omitempty"` + Configurations []description `json:"configurations,omitempty"` + References []reference `json:"references,omitempty"` + Timeline timeline `json:"timeline,omitempty"` + Credits credits `json:"credits,omitempty"` + Source json.RawMessage `json:"source,omitempty"` + Tags []string `json:"tags,omitempty"` + TaxonomyMappings taxonomyMappings `json:"taxonomyMappings,omitempty"` + DateAssigned *string `json:"dateAssigned,omitempty"` + DatePublic *string `json:"datePublic,omitempty"` + RejectedReasons []description `json:"rejectedReasons,omitempty"` + ReplacedBy []string `json:"replacedBy,omitempty"` + XGenerator interface{} `json:"x_generator,omitempty"` + XLegacyV4Record interface{} `json:"x_legacyV4Record,omitempty"` + XRedhatCweChain interface{} `json:"x_redhatCweChain,omitempty"` + XConverterErrors interface{} `json:"x_ConverterErrors,omitempty"` +} + +type adp struct { + ProviderMetadata providerMetadata `json:"providerMetadata"` + Title *string `json:"title,omitempty"` + Descriptions []description `json:"descriptions,omitempty"` + Affected []product `json:"affected,omitempty"` + ProblemTypes []problemType `json:"problemTypes,omitempty"` + Impacts []impact `json:"impacts,omitempty"` + Metrics []metric `json:"metrics,omitempty"` + Workarounds []description `json:"workarounds,omitempty"` + Solutions []description `json:"solutions,omitempty"` + Exploits []description `json:"exploits,omitempty"` + Configurations []description `json:"configurations,omitempty"` + References []reference `json:"references,omitempty"` + Timeline timeline `json:"timeline,omitempty"` + Credits credits `json:"credits,omitempty"` + Source json.RawMessage `json:"source,omitempty"` + Tags []string `json:"tags,omitempty"` + TaxonomyMappings taxonomyMappings `json:"taxonomyMappings,omitempty"` + DatePublic *string `json:"datePublic,omitempty"` +} + +type providerMetadata struct { + OrgID string `json:"orgID"` + ShortName *string `json:"shortName,omitempty"` + DateUpdated *string `json:"dateUpdated,omitempty"` +} + +type description struct { + Lang string `json:"lang"` + Value string `json:"value"` + SupportingMedia []struct { + Type string `json:"type"` + Base64 *bool `json:"base64,omitempty"` + Value string `json:"value"` + } `json:"supportingMedia,omitempty"` +} + +type product struct { + Vendor *string `json:"vendor,omitempty"` + Product *string `json:"product,omitempty"` + CollectionURL *string `json:"collectionURL,omitempty"` + PackageName *string `json:"packageName,omitempty"` + Cpes []string `json:"cpes,omitempty"` + Modules []string `json:"modules,omitempty"` + ProgramFiles []string `json:"programFiles,omitempty"` + ProgramRoutines []struct { + Name string `json:"name"` + } `json:"programRoutines,omitempty"` + Platforms []string `json:"platforms,omitempty"` + Repo *string `json:"repo,omitempty"` + DefaultStatus *string `json:"defaultStatus,omitempty"` + Versions []struct { + Status string `json:"status"` + VersionType *string `json:"versionType,omitempty"` + Version string `json:"version"` + LessThan *string `json:"lessThan,omitempty"` + LessThanOrEqual *string `json:"lessThanOrEqual,omitempty"` + Changes []struct { + At string `json:"at"` + Status string `json:"status"` + } `json:"changes,omitempty"` + } `json:"versions,omitempty"` +} + +type problemType struct { + Descriptions []struct { + Type *string `json:"type,omitempty"` + Lang string `json:"lang"` + Description string `json:"description"` + CweID *string `json:"cweId,omitempty"` + References []reference `json:"references,omitempty"` + } `json:"descriptions"` +} + +type impact struct { + Descriptions []description `json:"descriptions"` + CapecID *string `json:"capecId,omitempty"` +} + +type metric struct { + Format *string `json:"format,omitempty"` + Scenarios []struct { + Lang string `json:"lang"` + Value string `json:"value"` + } `json:"scenarios,omitempty"` + CVSSv2 *cvssv2 `json:"cvssV2_0,omitempty"` + CVSSv30 *cvssv30 `json:"cvssV3_0,omitempty"` + CVSSv31 *cvssv31 `json:"cvssV3_1,omitempty"` + CVSSv40 *cvssv40 `json:"cvssV4_0,omitempty"` + Other *struct { + Type string `json:"type"` + Content json.RawMessage `json:"content"` + } `json:"other,omitempty"` +} + +type cvssv2 struct { + Version string `json:"version"` + VectorString string `json:"vectorString"` + AccessVector *string `json:"accessVector,omitempty"` + AccessComplexity *string `json:"accessComplexity,omitempty"` + Authentication *string `json:"authentication,omitempty"` + ConfidentialityImpact *string `json:"confidentialityImpact,omitempty"` + IntegrityImpact *string `json:"integrityImpact,omitempty"` + AvailabilityImpact *string `json:"availabilityImpact,omitempty"` + BaseScore float64 `json:"baseScore"` + Exploitability *string `json:"exploitability,omitempty"` + RemediationLevel *string `json:"remediationLevel,omitempty"` + ReportConfidence *string `json:"reportConfidence,omitempty"` + TemporalScore *float64 `json:"temporalScore,omitempty"` + CollateralDamagePotential *string `json:"collateralDamagePotential,omitempty"` + TargetDistribution *string `json:"targetDistribution,omitempty"` + ConfidentialityRequirement *string `json:"confidentialityRequirement,omitempty"` + IntegrityRequirement *string `json:"integrityRequirement,omitempty"` + AvailabilityRequirement *string `json:"availabilityRequirement,omitempty"` + EnvironmentalScore *float64 `json:"environmentalScore,omitempty"` +} + +type cvssv30 struct { + Version string `json:"version"` + VectorString string `json:"vectorString"` + AttackVector *string `json:"attackVector,omitempty"` + AttackComplexity *string `json:"attackComplexity,omitempty"` + PrivilegesRequired *string `json:"privilegesRequired,omitempty"` + UserInteraction *string `json:"userInteraction,omitempty"` + Scope *string `json:"scope,omitempty"` + ConfidentialityImpact *string `json:"confidentialityImpact,omitempty"` + IntegrityImpact *string `json:"integrityImpact,omitempty"` + AvailabilityImpact *string `json:"availabilityImpact,omitempty"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` + ExploitCodeMaturity *string `json:"exploitCodeMaturity,omitempty"` + RemediationLevel *string `json:"remediationLevel,omitempty"` + ReportConfidence *string `json:"reportConfidence,omitempty"` + TemporalScore *float64 `json:"temporalScore,omitempty"` + TemporalSeverity *string `json:"temporalSeverity,omitempty"` + ConfidentialityRequirement *string `json:"confidentialityRequirement,omitempty"` + IntegrityRequirement *string `json:"integrityRequirement,omitempty"` + AvailabilityRequirement *string `json:"availabilityRequirement,omitempty"` + ModifiedAttackVector *string `json:"modifiedAttackVector,omitempty"` + ModifiedAttackComplexity *string `json:"modifiedAttackComplexity,omitempty"` + ModifiedPrivilegesRequired *string `json:"modifiedPrivilegesRequired,omitempty"` + ModifiedUserInteraction *string `json:"modifiedUserInteraction,omitempty"` + ModifiedScope *string `json:"modifiedScope,omitempty"` + ModifiedConfidentialityImpact *string `json:"modifiedConfidentialityImpact,omitempty"` + ModifiedIntegrityImpact *string `json:"modifiedIntegrityImpact,omitempty"` + ModifiedAvailabilityImpact *string `json:"modifiedAvailabilityImpact,omitempty"` + EnvironmentalScore *float64 `json:"environmentalScore,omitempty"` + EnvironmentalSeverity *string `json:"environmentalSeverity,omitempty"` +} + +type cvssv31 struct { + Version string `json:"version"` + VectorString string `json:"vectorString"` + AttackVector *string `json:"attackVector,omitempty"` + AttackComplexity *string `json:"attackComplexity,omitempty"` + PrivilegesRequired *string `json:"privilegesRequired,omitempty"` + UserInteraction *string `json:"userInteraction,omitempty"` + Scope *string `json:"scope,omitempty"` + ConfidentialityImpact *string `json:"confidentialityImpact,omitempty"` + IntegrityImpact *string `json:"integrityImpact,omitempty"` + AvailabilityImpact *string `json:"availabilityImpact,omitempty"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` + ExploitCodeMaturity *string `json:"exploitCodeMaturity,omitempty"` + RemediationLevel *string `json:"remediationLevel,omitempty"` + ReportConfidence *string `json:"reportConfidence,omitempty"` + TemporalScore *float64 `json:"temporalScore,omitempty"` + TemporalSeverity *string `json:"temporalSeverity,omitempty"` + ConfidentialityRequirement *string `json:"confidentialityRequirement,omitempty"` + IntegrityRequirement *string `json:"integrityRequirement,omitempty"` + AvailabilityRequirement *string `json:"availabilityRequirement,omitempty"` + ModifiedAttackVector *string `json:"modifiedAttackVector,omitempty"` + ModifiedAttackComplexity *string `json:"modifiedAttackComplexity,omitempty"` + ModifiedPrivilegesRequired *string `json:"modifiedPrivilegesRequired,omitempty"` + ModifiedUserInteraction *string `json:"modifiedUserInteraction,omitempty"` + ModifiedScope *string `json:"modifiedScope,omitempty"` + ModifiedConfidentialityImpact *string `json:"modifiedConfidentialityImpact,omitempty"` + ModifiedIntegrityImpact *string `json:"modifiedIntegrityImpact,omitempty"` + ModifiedAvailabilityImpact *string `json:"modifiedAvailabilityImpact,omitempty"` + EnvironmentalScore *float64 `json:"environmentalScore,omitempty"` + EnvironmentalSeverity *string `json:"environmentalSeverity,omitempty"` +} + +type cvssv40 struct { + Version string `json:"version"` + VectorString string `json:"vectorString"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` + AttackVector *string `json:"attackVector,omitempty"` + AttackComplexity *string `json:"attackComplexity,omitempty"` + AttackRequirements *string `json:"attackRequirements,omitempty"` + PrivilegesRequired *string `json:"privilegesRequired,omitempty"` + UserInteraction *string `json:"userInteraction,omitempty"` + VulnConfidentialityImpact *string `json:"vulnConfidentialityImpact,omitempty"` + VulnIntegrityImpact *string `json:"vulnIntegrityImpact,omitempty"` + VulnAvailabilityImpact *string `json:"vulnAvailabilityImpact,omitempty"` + SubConfidentialityImpact *string `json:"subConfidentialityImpact,omitempty"` + SubIntegrityImpact *string `json:"subIntegrityImpact,omitempty"` + SubAvailabilityImpact *string `json:"subAvailabilityImpact,omitempty"` + ExploitMaturity *string `json:"exploitMaturity,omitempty"` + ConfidentialityRequirement *string `json:"confidentialityRequirement,omitempty"` + IntegrityRequirement *string `json:"integrityRequirement,omitempty"` + AvailabilityRequirement *string `json:"availabilityRequirement,omitempty"` + ModifiedAttackVector *string `json:"modifiedAttackVector,omitempty"` + ModifiedAttackComplexity *string `json:"modifiedAttackComplexity,omitempty"` + ModifiedAttackRequirements *string `json:"modifiedAttackRequirements,omitempty"` + ModifiedPrivilegesRequired *string `json:"modifiedPrivilegesRequired,omitempty"` + ModifiedUserInteraction *string `json:"modifiedUserInteraction,omitempty"` + ModifiedVulnConfidentialityImpact *string `json:"modifiedVulnConfidentialityImpact,omitempty"` + ModifiedVulnIntegrityImpact *string `json:"modifiedVulnIntegrityImpact,omitempty"` + ModifiedVulnAvailabilityImpact *string `json:"modifiedVulnAvailabilityImpact,omitempty"` + ModifiedSubConfidentialityImpact *string `json:"modifiedSubConfidentialityImpact,omitempty"` + ModifiedSubIntegrityImpact *string `json:"modifiedSubIntegrityImpact,omitempty"` + ModifiedSubAvailabilityImpact *string `json:"modifiedSubAvailabilityImpact,omitempty"` + Safety *string `json:"Safety,omitempty"` + Automatable *string `json:"Automatable,omitempty"` + Recovery *string `json:"Recovery,omitempty"` + ValueDensity *string `json:"valueDensity,omitempty"` + VulnerabilityResponseEffort *string `json:"vulnerabilityResponseEffort,omitempty"` + ProviderUrgency *string `json:"providerUrgency,omitempty"` + ThreatScore *float64 `json:"threatScore,omitempty"` + ThreatSeverity *string `json:"threatSeverity,omitempty"` + EnvironmentalScore *float64 `json:"environmentalScore,omitempty"` + EnvironmentalSeverity *string `json:"environmentalSeverity,omitempty"` +} + +type ssvc struct { + ID string `json:"id,omitempty"` + Role string `json:"role,omitempty"` + Version string `json:"version,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + Options []struct { + Exploitation *string `json:"Exploitation,omitempty"` + Automatable *string `json:"Automatable,omitempty"` + TechnicalImpact *string `json:"Technical Impact,omitempty"` + } `json:"options,omitempty"` +} + +type kev struct { + DateAdded string `json:"date_added"` + Reference string `json:"reference"` +} + +type reference struct { + Name *string `json:"name,omitempty"` + Tags []string `json:"tags,omitempty"` + URL string `json:"url"` +} + +type timeline []struct { + Time string `json:"time"` + Lang string `json:"lang"` + Value string `json:"value"` +} + +type credits []struct { + Type *string `json:"type,omitempty"` + Lang string `json:"lang"` + User *string `json:"user,omitempty"` + Value string `json:"value"` +} + +type taxonomyMappings []struct { + TaxonomyVersion *string `json:"taxonomyVersion,omitempty"` + TaxonomyName string `json:"taxonomyName"` + TaxonomyRelations []struct { + TaxonomyID string `json:"taxonomyId"` + RelationshipName string `json:"relationshipName"` + RelationshipValue string `json:"relationshipValue"` + } `json:"taxonomyRelations"` +} diff --git a/models/models.go b/models/models.go index e0131edc..adb8b0db 100644 --- a/models/models.go +++ b/models/models.go @@ -1,7 +1,6 @@ package models import ( - "fmt" "time" "gorm.io/gorm" @@ -14,6 +13,8 @@ const ( JvnType = "JVN" // FortinetType : FortinetType = "Fortinet" + // MitreType : + MitreType = "Mitre" // NvdExactVersionMatch : NvdExactVersionMatch = "NvdExactVersionMatch" @@ -29,27 +30,14 @@ const ( FortinetRoughVersionMatch = "FortinetRoughVersionMatch" // FortinetVendorProductMatch : FortinetVendorProductMatch = "FortinetVendorProductMatch" + // MitreExactVersionMatch : + MitreExactVersionMatch = "MitreExactVersionMatch" + // MitreRoughVersionMatch : + MitreRoughVersionMatch = "MitreRoughVersionMatch" + // MitreVendorProductMatch : + MitreVendorProductMatch = "MitreVendorProductMatch" ) -// GetURLByYear returns url -func GetURLByYear(source, year string) (url string, err error) { - switch source { - case NvdType: - return fmt.Sprintf("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%s.json.gz", year), nil - case JvnType: - switch year { - case "modified": - return "https://jvndb.jvn.jp/ja/rss/jvndb.rdf", nil - case "recent": - return "https://jvndb.jvn.jp/ja/rss/jvndb_new.rdf", nil - default: - return fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/years/jvndb_%s.rdf", year), nil - } - default: - return "", fmt.Errorf("unknown source type: %s", source) - } -} - // LatestSchemaVersion manages the Schema version used in the latest go-cve-dictionary. const LatestSchemaVersion = 3 @@ -72,6 +60,7 @@ type CveDetail struct { Nvds []Nvd Jvns []Jvn Fortinets []Fortinet + Mitres []Mitre } // HasNvd returns true if NVD contents @@ -89,11 +78,17 @@ func (c CveDetail) HasFortinet() bool { return len(c.Fortinets) != 0 } +// HasMitre returns true if Mitre contents +func (c CveDetail) HasMitre() bool { + return len(c.Mitres) != 0 +} + // CpeDetail : type CpeDetail struct { Nvds []NvdCpe Jvns []JvnCpe Fortinet []FortinetCpe + // Mitre []MitreCpe } // Components common to Nvd and Jvn @@ -128,6 +123,17 @@ type Cvss3 struct { ImpactScore float64 } +// Cvss40 has CVSS Version 4.0 info +type Cvss40 struct { + VectorString string `gorm:"type:varchar(255)"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `gorm:"type:varchar(255)"` + ThreatScore *float64 + ThreatSeverity *string `gorm:"type:varchar(255)"` + EnvironmentalScore *float64 + EnvironmentalSeverity *string `gorm:"type:varchar(255)"` +} + // CpeBase has common args of Cpe and EnvCpe type CpeBase struct { URI string `gorm:"index;type:varchar(255)"` @@ -222,7 +228,7 @@ type NvdCvss3 struct { // NvdCwe has CweID type NvdCwe struct { ID int64 `json:"-"` - NvdID uint `json:"-" index:"idx_nvd_cwes_nvd_id"` + NvdID uint `json:"-" gorm:"index:idx_nvd_cwes_nvd_id"` Source string `gorm:"type:text"` Type string `gorm:"type:varchar(255)"` CweID string `gorm:"type:varchar(255)"` @@ -346,7 +352,7 @@ type FortinetCvss3 struct { // FortinetCwe has CweID type FortinetCwe struct { ID int64 `json:"-"` - FortinetID uint `json:"-" index:"idx_fortinet_cwes_fortinet_id"` + FortinetID uint `json:"-" gorm:"index:idx_fortinet_cwes_fortinet_id"` CweID string `gorm:"type:varchar(255)"` } @@ -363,3 +369,341 @@ type FortinetReference struct { FortinetID uint `json:"-" gorm:"index:idx_fortinet_references_fortinet_id"` Reference `gorm:"embedded"` } + +// Mitre : https://cveproject.github.io/cve-schema/schema/CVE_Record_Format.json +type Mitre struct { + ID int64 `json:"-"` + DataType string `gorm:"type:varchar(255)"` + DataVersion string `gorm:"type:varchar(255)"` + CVEMetadata MitreCVEMetadata + Containers []MitreContainer +} + +// MitreCVEMetadata : #/definitions/cveMetadataPublished +type MitreCVEMetadata struct { + ID int64 `json:"-"` + MitreID uint `json:"-" gorm:"index:idx_mitre_cve_metadata"` + CVEID string `gorm:"index:idx_mitre_cve_metadata_cveid;type:varchar(255)"` + AssignerOrgID string `gorm:"type:varchar(255)"` + AssignerShortName *string `gorm:"type:varchar(32)"` + RequesterUserID *string `gorm:"type:varchar(255)"` + Serial *int + State string `gorm:"type:varchar(255)"` + DatePublished *time.Time + DateUpdated *time.Time + DateReserved *time.Time + DateRejected *time.Time +} + +// MitreContainer : #/definitions/cnaPublishedContainer, #/definitions/adpContainer +type MitreContainer struct { + ID int64 `json:"-"` + MitreID uint `json:"-" gorm:"index:idx_mitre_containers"` + ContainerType string `gorm:"type:varchar(255)"` + ProviderMetadata MitreProviderMetadata + Title *string `gorm:"type:varchar(256)"` + Descriptions []MitreDescription + // Affected []MitreCpe + ProblemTypes []MitreProblemType + Impacts []MitreImpact + Metrics []MitreMetric + Workarounds []MitreWorkaround + Solutions []MitreSolution + Exploits []MitreExploit + Configurations []MitreConfiguration + References []MitreReference + Timeline []MitreTimeline + Credits []MitreCredit + Source string `gorm:"type:text"` + Tags []MitreTag + TaxonomyMappings []MitreTaxonomyMapping + DateAssigned *time.Time + DatePublic *time.Time +} + +// MitreProviderMetadata : #/definitions/providerMetadata +type MitreProviderMetadata struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_provider_metadata"` + OrgID string `gorm:"type:varchar(255)"` + ShortName *string `gorm:"type:varchar(32)"` + DateUpdated *time.Time +} + +// MitreDescription : #/definitions/description +type MitreDescription struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_description"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreDescriptionSupportingMedia +} + +// MitreDescriptionSupportingMedia : #/definitions/description +type MitreDescriptionSupportingMedia struct { + ID int64 `json:"-"` + MitreDescriptionID uint `json:"-" gorm:"index:idx_mitre_description_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreCpe : +// type MitreCpe struct { +// ID int64 `json:"-"` +// MitreContainerID uint `json:"-" gorm:"index:idx_mitre_cpe"` +// CpeBase `gorm:"embedded"` +// } + +// MitreProblemType : #/definitions/problemTypes +type MitreProblemType struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_problem_type"` + Descriptions []MitreProblemTypeDescription +} + +// MitreProblemTypeDescription : #/definitions/problemTypes +type MitreProblemTypeDescription struct { + ID int64 `json:"-"` + MitreProblemTypeID uint `json:"-" gorm:"index:idx_mitre_problem_type_description"` + Type *string `gorm:"type:varchar(255)"` + Lang string `gorm:"type:varchar(128)"` + Description string `gorm:"type:text"` + CweID *string `gorm:"type:varchar(9)"` + References []MitreProblemTypeDescriptionReference +} + +// MitreProblemTypeDescriptionReference : #/definitions/references +type MitreProblemTypeDescriptionReference struct { + ID int64 `json:"-"` + MitreProblemTypeDescriptionID uint `json:"-" gorm:"index:idx_mitre_problem_type_description_reference"` + Reference `gorm:"embedded"` +} + +// MitreImpact : #/definitions/impacts +type MitreImpact struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_impact"` + Descriptions []MitreImpactDescription + CapecID *string `gorm:"type:varchar(11)"` +} + +// MitreImpactDescription : #/definitions/description +type MitreImpactDescription struct { + ID int64 `json:"-"` + MitreImpactID uint `json:"-" gorm:"index:idx_mitre_impact_description"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreImpactDescriptionSupportingMedia +} + +// MitreImpactDescriptionSupportingMedia : #/definitions/description +type MitreImpactDescriptionSupportingMedia struct { + ID int64 `json:"-"` + MitreImpactDescriptionID uint `json:"-" gorm:"index:idx_mitre_impact_description_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreMetric : #/definitions/metrics +type MitreMetric struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_metric"` + Format string `gorm:"type:varchar(64)"` + Scenarios []MitreMetricScenario + CVSSv2 *MitreMetricCVSS2 + CVSSv30 *MitreMetricCVSS30 + CVSSv31 *MitreMetricCVSS31 + CVSSv40 *MitreMetricCVSS40 + SSVC *MitreMetricSSVC + KEV *MitreMetricKEV + Other *MitreMetricOther +} + +// MitreMetricScenario : #/definitions/metrics +type MitreMetricScenario struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_scenario"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` +} + +// MitreMetricCVSS2 : https://www.first.org/cvss/cvss-v2.0.json?20170531 +type MitreMetricCVSS2 struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_cvss2"` + Cvss2 `gorm:"embedded"` +} + +// MitreMetricCVSS30 : https://www.first.org/cvss/cvss-v3.0.json?20170531 +type MitreMetricCVSS30 struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_cvss30"` + Cvss3 `gorm:"embedded"` +} + +// MitreMetricCVSS31 : https://www.first.org/cvss/cvss-v3.1.json?20210501 +type MitreMetricCVSS31 struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_cvss31"` + Cvss3 `gorm:"embedded"` +} + +// MitreMetricCVSS40 : https://www.first.org/cvss/cvss-v4.0.json?20231011 +type MitreMetricCVSS40 struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_cvss40"` + Cvss40 `gorm:"embedded"` +} + +// MitreMetricSSVC : +type MitreMetricSSVC struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_ssvc"` + Role string `gorm:"type:varchar(255)"` + Version string `gorm:"type:varchar(255)"` + Timestamp time.Time + Exploitation *string `gorm:"type:varchar(255)"` + Automatable *string `gorm:"type:varchar(255)"` + TechnicalImpact *string `gorm:"type:varchar(255)"` +} + +// MitreMetricKEV : https://github.com/cisagov/vulnrichment/blob/3f9d69632037fae3b7abdf47fc848c287702ffaa/assets/kev_metrics_schema-1.0.json +type MitreMetricKEV struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_kev"` + DateAdded time.Time + Reference string `gorm:"type:varchar(255)"` +} + +// MitreMetricOther : https://github.com/CVEProject/cve-schema/blob/30f59c7de92fbc77bddade302601cb500c66f718/schema/CVE_Record_Format.json#L901-L923 +type MitreMetricOther struct { + ID int64 `json:"-"` + MitreMetricID uint `json:"-" gorm:"index:idx_mitre_metric_other"` + Type string `gorm:"type:varchar(128)"` + Content string `gorm:"type:text"` +} + +// MitreWorkaround : #/definitions/workarounds +type MitreWorkaround struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_workaround"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreWorkaroundSupportingMedia +} + +// MitreWorkaroundSupportingMedia : #/definitions/description +type MitreWorkaroundSupportingMedia struct { + ID int64 `json:"-"` + MitreWorkaroundID uint `json:"-" gorm:"index:idx_mitre_workaround_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreSolution : #/definitions/solutions +type MitreSolution struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_solution"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreSolutionSupportingMedia +} + +// MitreSolutionSupportingMedia : #/definitions/description +type MitreSolutionSupportingMedia struct { + ID int64 `json:"-"` + MitreSolutionID uint `json:"-" gorm:"index:idx_mitre_solution_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreExploit : #/definitions/exploits +type MitreExploit struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_exploit"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreExploitSupportingMedia +} + +// MitreExploitSupportingMedia : #/definitions/description +type MitreExploitSupportingMedia struct { + ID int64 `json:"-"` + MitreExploitID uint `json:"-" gorm:"index:idx_mitre_exploit_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreConfiguration : #/definitions/configurations +type MitreConfiguration struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_configuration"` + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` + SupportingMedia []MitreConfigurationSupportingMedia +} + +// MitreConfigurationSupportingMedia : #/definitions/description +type MitreConfigurationSupportingMedia struct { + ID int64 `json:"-"` + MitreConfigurationID uint `json:"-" gorm:"index:idx_mitre_configuration_supporting_media"` + Type string `gorm:"type:varchar(256)"` + Base64 *bool + Value string `gorm:"type:text"` +} + +// MitreReference : #/definitions/references +type MitreReference struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_reference"` + Reference `gorm:"embedded"` +} + +// MitreTimeline : #/definitions/timeline +type MitreTimeline struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_timeline"` + Time time.Time + Lang string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` +} + +// MitreCredit : #/definitions/credits +type MitreCredit struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_credit"` + Type *string `gorm:"type:varchar(255)"` + Lang string `gorm:"type:varchar(255)"` + User *string `gorm:"type:varchar(255)"` + Value string `gorm:"type:text"` +} + +// MitreTag : #/definitions/cnaTags, #/definitions/adpTags +type MitreTag struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_tag"` + Tag string `gorm:"type:varchar(255)"` +} + +// MitreTaxonomyMapping : #/definitions/taxonomyMappings +type MitreTaxonomyMapping struct { + ID int64 `json:"-"` + MitreContainerID uint `json:"-" gorm:"index:idx_mitre_taxonomy_mapping"` + TaxonomyVersion *string `gorm:"type:varchar(128)"` + TaxonomyName string `gorm:"type:varchar(128)"` + TaxonomyRelations []MitreTaxonomyRelation +} + +// MitreTaxonomyRelation : #/definitions/taxonomyMappings +type MitreTaxonomyRelation struct { + ID int64 `json:"-"` + MitreTaxonomyMappingID uint `json:"-" gorm:"index:idx_mitre_taxonomy_relation"` + TaxonomyID string `gorm:"type:text"` + RelationshipName string `gorm:"type:varchar(128)"` + RelationshipValue string `gorm:"type:text"` +} diff --git a/models/models_test.go b/models/models_test.go index 683cc631..89dda0ac 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -4,63 +4,6 @@ import ( "testing" ) -func Test_GetURLByYear(t *testing.T) { - type args struct { - source string - year string - } - var tests = []struct { - in args - url string - wantErr bool - }{ - { - in: args{ - source: NvdType, - year: "recent", - }, - url: "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-recent.json.gz", - }, - { - in: args{ - source: JvnType, - year: "recent", - }, - url: "https://jvndb.jvn.jp/ja/rss/jvndb_new.rdf", - }, - { - in: args{ - source: JvnType, - year: "modified", - }, - url: "https://jvndb.jvn.jp/ja/rss/jvndb.rdf", - }, - { - in: args{ - source: JvnType, - year: "2021", - }, - url: "https://jvndb.jvn.jp/ja/rss/years/jvndb_2021.rdf", - }, - { - in: args{ - source: "unknown", - }, - wantErr: true, - }, - } - for i, tt := range tests { - actual, err := GetURLByYear(tt.in.source, tt.in.year) - if (err != nil) != tt.wantErr { - t.Errorf("[%d] GetURLByYear(%s, %s) error = %v, wantErr %v", i, tt.in.source, tt.in.year, err, tt.wantErr) - continue - } - if actual != tt.url { - t.Errorf("[%d] GetURLByYear(%s, %s) actual = %v, expected = %v", i, tt.in.source, tt.in.year, actual, tt.url) - } - } -} - func Test_FetchMeta(t *testing.T) { var tests = []struct { in FetchMeta