Skip to content

Commit

Permalink
Ent: IngestBuilders, IngestCertifyBads, IngestCertifyGoods, IngestCer…
Browse files Browse the repository at this point in the history 
…tifyLegals refactored concurrently (#1599)

Signed-off-by: mrizzi <[email protected]>
  • Loading branch information
@mrizzi
mrizzi authored Dec 22, 2023
1 parent c5d84b6 commit f6a0a24
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 36 deletions.
24 changes: 16 additions & 8 deletions pkg/assembler/backends/ent/backend/builders.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"github.com/guacsec/guac/pkg/assembler/backends/ent/predicate"
"github.com/guacsec/guac/pkg/assembler/graphql/model"
"github.com/pkg/errors"
"github.com/vektah/gqlparser/v2/gqlerror"
"golang.org/x/sync/errgroup"
)

func (b *EntBackend) Builders(ctx context.Context, builderSpec *model.BuilderSpec) ([]*model.Builder, error) {
Expand Down Expand Up @@ -71,13 +71,21 @@ func (b *EntBackend) IngestBuilder(ctx context.Context, build *model.BuilderInpu
}

func (b *EntBackend) IngestBuilders(ctx context.Context, builders []*model.BuilderInputSpec) ([]string, error) {
var buildersID []string
for _, builder := range builders {
id, err := b.IngestBuilder(ctx, builder)
if err != nil {
return nil, gqlerror.Errorf("IngestBuilders failed with err: %v", err)
}
buildersID = append(buildersID, id)
buildersID := make([]string, len(builders))
eg, ctx := errgroup.WithContext(ctx)
for i := range builders {
index := i
bld := builders[index]
concurrently(eg, func() error {
id, err := b.IngestBuilder(ctx, bld)
if err == nil {
buildersID[index] = id
}
return err
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return buildersID, nil
}
Expand Down
54 changes: 35 additions & 19 deletions pkg/assembler/backends/ent/backend/certify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"github.com/guacsec/guac/pkg/assembler/backends/ent/certification"
"github.com/guacsec/guac/pkg/assembler/backends/ent/predicate"
"github.com/guacsec/guac/pkg/assembler/graphql/model"
"github.com/vektah/gqlparser/v2/gqlerror"
"golang.org/x/sync/errgroup"
)

type certificationInputSpec interface {
Expand Down Expand Up @@ -67,21 +67,29 @@ func (b *EntBackend) IngestCertifyBad(ctx context.Context, subject model.Package
}

func (b *EntBackend) IngestCertifyBads(ctx context.Context, subjects model.PackageSourceOrArtifactInputs, pkgMatchType *model.MatchFlags, certifyBads []*model.CertifyBadInputSpec) ([]string, error) {
var result []string
result := make([]string, len(certifyBads))
eg, ctx := errgroup.WithContext(ctx)
for i := range certifyBads {
index := i
var subject model.PackageSourceOrArtifactInput
if len(subjects.Packages) > 0 {
subject = model.PackageSourceOrArtifactInput{Package: subjects.Packages[i]}
subject = model.PackageSourceOrArtifactInput{Package: subjects.Packages[index]}
} else if len(subjects.Artifacts) > 0 {
subject = model.PackageSourceOrArtifactInput{Artifact: subjects.Artifacts[i]}
subject = model.PackageSourceOrArtifactInput{Artifact: subjects.Artifacts[index]}
} else {
subject = model.PackageSourceOrArtifactInput{Source: subjects.Sources[i]}
subject = model.PackageSourceOrArtifactInput{Source: subjects.Sources[index]}
}
cb, err := b.IngestCertifyBad(ctx, subject, pkgMatchType, *certifyBads[i])
if err != nil {
return nil, gqlerror.Errorf("IngestCertifyBads failed with err: %v", err)
}
result = append(result, cb)
certifyBad := *certifyBads[index]
concurrently(eg, func() error {
cb, err := b.IngestCertifyBad(ctx, subject, pkgMatchType, certifyBad)
if err == nil {
result[index] = cb
}
return err
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return result, nil
}
Expand All @@ -100,21 +108,29 @@ func (b *EntBackend) IngestCertifyGood(ctx context.Context, subject model.Packag
}

func (b *EntBackend) IngestCertifyGoods(ctx context.Context, subjects model.PackageSourceOrArtifactInputs, pkgMatchType *model.MatchFlags, certifyGoods []*model.CertifyGoodInputSpec) ([]string, error) {
var result []string
result := make([]string, len(certifyGoods))
eg, ctx := errgroup.WithContext(ctx)
for i := range certifyGoods {
index := i
var subject model.PackageSourceOrArtifactInput
if len(subjects.Packages) > 0 {
subject = model.PackageSourceOrArtifactInput{Package: subjects.Packages[i]}
subject = model.PackageSourceOrArtifactInput{Package: subjects.Packages[index]}
} else if len(subjects.Artifacts) > 0 {
subject = model.PackageSourceOrArtifactInput{Artifact: subjects.Artifacts[i]}
subject = model.PackageSourceOrArtifactInput{Artifact: subjects.Artifacts[index]}
} else {
subject = model.PackageSourceOrArtifactInput{Source: subjects.Sources[i]}
subject = model.PackageSourceOrArtifactInput{Source: subjects.Sources[index]}
}
cg, err := b.IngestCertifyGood(ctx, subject, pkgMatchType, *certifyGoods[i])
if err != nil {
return nil, gqlerror.Errorf("IngestCertifyGoods failed with err: %v", err)
}
result = append(result, cg)
certifyGood := *certifyGoods[index]
concurrently(eg, func() error {
cg, err := b.IngestCertifyGood(ctx, subject, pkgMatchType, certifyGood)
if err == nil {
result[index] = cg
}
return err
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return result, nil
}
Expand Down
27 changes: 19 additions & 8 deletions pkg/assembler/backends/ent/backend/certifyLegal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/guacsec/guac/pkg/assembler/graphql/model"
"github.com/pkg/errors"
"github.com/vektah/gqlparser/v2/gqlerror"
"golang.org/x/sync/errgroup"
)

func (b *EntBackend) CertifyLegal(ctx context.Context, spec *model.CertifyLegalSpec) ([]*model.CertifyLegal, error) {
Expand Down Expand Up @@ -69,19 +70,29 @@ func (b *EntBackend) CertifyLegal(ctx context.Context, spec *model.CertifyLegalS
}

func (b *EntBackend) IngestCertifyLegals(ctx context.Context, subjects model.PackageOrSourceInputs, declaredLicensesList [][]*model.LicenseInputSpec, discoveredLicensesList [][]*model.LicenseInputSpec, certifyLegals []*model.CertifyLegalInputSpec) ([]string, error) {
var modelCertifyLegals []string
modelCertifyLegals := make([]string, len(certifyLegals))
eg, ctx := errgroup.WithContext(ctx)
for i := range certifyLegals {
index := i
var subject model.PackageOrSourceInput
if len(subjects.Packages) > 0 {
subject = model.PackageOrSourceInput{Package: subjects.Packages[i]}
subject = model.PackageOrSourceInput{Package: subjects.Packages[index]}
} else {
subject = model.PackageOrSourceInput{Source: subjects.Sources[i]}
subject = model.PackageOrSourceInput{Source: subjects.Sources[index]}
}
modelCertifyLegal, err := b.IngestCertifyLegal(ctx, subject, declaredLicensesList[i], discoveredLicensesList[i], certifyLegals[i])
if err != nil {
return nil, gqlerror.Errorf("IngestCertifyLegal failed with element #%v with err: %v", i, err)
}
modelCertifyLegals = append(modelCertifyLegals, modelCertifyLegal)
declaredLicense := declaredLicensesList[index]
discoveredLicense := discoveredLicensesList[index]
certifyLegal := certifyLegals[index]
concurrently(eg, func() error {
modelCertifyLegal, err := b.IngestCertifyLegal(ctx, subject, declaredLicense, discoveredLicense, certifyLegal)
if err == nil {
modelCertifyLegals[index] = modelCertifyLegal
}
return err
})
}
if err := eg.Wait(); err != nil {
return nil, err
}
return modelCertifyLegals, nil
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/assembler/backends/ent/backend/certifyLegal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -608,7 +608,7 @@ func (s *Suite) TestLegals() {
if err != nil {
return
}
if diff := cmp.Diff(test.ExpLegal, got, ignoreID); diff != "" {
if diff := cmp.Diff(test.ExpLegal, got, IngestPredicatesCmpOpts...); diff != "" {
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
}
})
Expand Down
28 changes: 28 additions & 0 deletions pkg/assembler/backends/ent/backend/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ var IngestPredicatesCmpOpts = []cmp.Option{
cmpopts.EquateEmpty(),
cmpopts.SortSlices(isDependencyLess),
cmpopts.SortSlices(packageLess),
cmpopts.SortSlices(sourceLess),
cmpopts.SortSlices(certifyVulnLess),
cmpopts.SortSlices(certifyVexLess),
cmpopts.SortSlices(vulnerabilityLess),
cmpopts.SortSlices(hasSbomLess),
cmpopts.SortSlices(certifyLegalLess),
}

func isDependencyLess(e1, e2 *model.IsDependency) bool {
Expand All @@ -66,6 +68,12 @@ func packageLess(e1, e2 *model.Package) bool {
return purl1 < purl2
}

func sourceLess(e1, e2 *model.Source) bool {
purl1 := helpers.PkgToPurl(e1.Type, e1.Namespaces[0].Namespace, e1.Namespaces[0].Names[0].Name, "", "", nil)
purl2 := helpers.PkgToPurl(e2.Type, e2.Namespaces[0].Namespace, e2.Namespaces[0].Names[0].Name, "", "", nil)
return purl1 < purl2
}

func certifyVulnLess(e1, e2 *model.CertifyVuln) bool {
return packageLess(e1.Package, e2.Package)
}
Expand Down Expand Up @@ -106,6 +114,26 @@ func hasSbomLess(e1, e2 *model.HasSbom) bool {
return false
}

func certifyLegalLess(e1, e2 *model.CertifyLegal) bool {
switch subject1 := e1.Subject.(type) {
case *model.Package:
switch subject2 := e2.Subject.(type) {
case *model.Package:
return packageLess(subject1, subject2)
case *model.Source:
return false
}
case *model.Source:
switch subject2 := e2.Subject.(type) {
case *model.Package:
return true
case *model.Source:
return sourceLess(subject1, subject2)
}
}
return false
}

func rewriteID(t *testing.T, queryID *string, realIDs []string) {
idIdx, err := strconv.Atoi(*queryID)
if err == nil {
Expand Down

0 comments on commit f6a0a24

Please sign in to comment.