Skip to content

Commit

Permalink
Update mutation API to return only ID (merge branch 1116-return-ID to…
Browse files Browse the repository at this point in the history
… main) (#1542)

* Backend interface - Methods for returning 'string' (#1285)

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Enable GitHub CI tests (#1296)

* Enable GitHub CI tests

Signed-off-by: mrizzi <[email protected]>

* Mocks backend updated

Signed-off-by: mrizzi <[email protected]>

* Ent - Bump v0.12.4-0.20230918073025-797534a0d1ca (#1283)

* Ent - Bump v0.12.4-0.20230918073025-797534a0d1ca

Signed-off-by: mrizzi <[email protected]>

* Ent - Pin postgres image to hash

Signed-off-by: mrizzi <[email protected]>

---------

Signed-off-by: mrizzi <[email protected]>

---------

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Ent - IngestArtifactID,IngestArtifactIDs implementations (#1300)

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Ent - IngestBuilderID, IngestPackageID, IngestSourceID, IngestVulnerabilityID (#1329)

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* Ent - upsertPackageIDDoNothing vs upsertPackageIDIgnore (#1270)

* Ent - upsertPackageIDDoNothing vs upsertPackageIDIgnore

Signed-off-by: mrizzi <[email protected]>

* Ent - final upsertPackage implementation

Signed-off-by: mrizzi <[email protected]>

---------

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: desmax74 <[email protected]>

* Ent - upsertPackage: enhanced ID management

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* Ent - Packages: subpath query issue  (#1330)

* Ent - Packages: subpath query issue reproducer

Signed-off-by: mrizzi <[email protected]>

* Ent - Packages: subpath query issue fix

Signed-off-by: mrizzi <[email protected]>

---------

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* Ent - TestIngestVulnerabilities changes to support ID

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* Ent - Node: added SourceName management

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

* 1116-return-ID

Signed-off-by: desmax74 <[email protected]>

---------

Signed-off-by: desmax74 <[email protected]>
Signed-off-by: mrizzi <[email protected]>
Co-authored-by: Marco Rizzi <[email protected]>
Co-authored-by: mrizzi <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Ent - IngestHasSBOMID (#1470)

Signed-off-by: desmax74 <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Ent - ScorecardID (#1469)

Signed-off-by: desmax74 <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Ent - PkgEqualsID (#1473)

Signed-off-by: desmax74 <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* update ent IngestHasSbomID and IngestHasSBOMIDs with the new includes field

Signed-off-by: pxp928 <[email protected]>

* Feature/1116 arango noun mutation update to return only IDs (#1511)

* change artifact mutation to return only ID

Signed-off-by: pxp928 <[email protected]>

* change builder and license to return only ID during ingestion

Signed-off-by: pxp928 <[email protected]>

* update package to return flat struct of IDs

Signed-off-by: pxp928 <[email protected]>

* update source and vulnerability to return mutation ID

Signed-off-by: pxp928 <[email protected]>

* update ENT for vulnerability and package mutation IDs

Signed-off-by: pxp928 <[email protected]>

* update ENT for source mutation IDs

Signed-off-by: pxp928 <[email protected]>

* update tests for arango and ent

Signed-off-by: pxp928 <[email protected]>

---------

Signed-off-by: pxp928 <[email protected]>

* Update all resolvers for Return ID (#1514)

* Update resolvers to use ID backend methods

Signed-off-by: Jeff Mendoza <[email protected]>

* Remove old methods from backend interface, update pkg/src/vuln resolvers.

Signed-off-by: Jeff Mendoza <[email protected]>

---------

Signed-off-by: Jeff Mendoza <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Update keyvalue backend to return ID (#1524)

* Update keyvalue backend to return ID

Signed-off-by: Jeff Mendoza <[email protected]>

* Lint gosimple

Signed-off-by: Jeff Mendoza <[email protected]>

* Regen backend mock

Signed-off-by: Jeff Mendoza <[email protected]>

---------

Signed-off-by: Jeff Mendoza <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* Feature/1116 arango verb mutation (#1529)

Signed-off-by: pxp928 <[email protected]>

* Complete Ent Return -ID (#1536)

* Updated ent with ID

Signed-off-by: Jeff Mendoza <[email protected]>

* Update ent tests for ID.

Signed-off-by: Jeff Mendoza <[email protected]>

---------

Signed-off-by: Jeff Mendoza <[email protected]>
Signed-off-by: pxp928 <[email protected]>

* change mutations back to original name (#1541)

Signed-off-by: pxp928 <[email protected]>

* remove branch from ci workflow

Signed-off-by: pxp928 <[email protected]>

---------

Signed-off-by: mrizzi <[email protected]>
Signed-off-by: pxp928 <[email protected]>
Signed-off-by: desmax74 <[email protected]>
Signed-off-by: Jeff Mendoza <[email protected]>
Co-authored-by: Marco Rizzi <[email protected]>
Co-authored-by: Massimiliano Dessì - (Fast Chauffeur) <[email protected]>
Co-authored-by: mrizzi <[email protected]>
Co-authored-by: Jeff Mendoza <[email protected]>
  • Loading branch information
5 people authored Nov 30, 2023
1 parent 686ce43 commit 32697ae
Show file tree
Hide file tree
Showing 197 changed files with 5,226 additions and 8,229 deletions.
152 changes: 76 additions & 76 deletions internal/testing/mocks/backend.go

Large diffs are not rendered by default.

35 changes: 18 additions & 17 deletions pkg/assembler/backends/arangodb/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func getArtifactQueryValues(artifact *model.ArtifactInputSpec) map[string]any {
return values
}

func (c *arangoClient) IngestArtifacts(ctx context.Context, artifacts []*model.ArtifactInputSpec) ([]*model.Artifact, error) {
func (c *arangoClient) IngestArtifacts(ctx context.Context, artifacts []*model.ArtifactInputSpec) ([]string, error) {
var listOfValues []map[string]any
for i := range artifacts {
listOfValues = append(listOfValues, getArtifactQueryValues(artifacts[i]))
Expand Down Expand Up @@ -206,11 +206,7 @@ func (c *arangoClient) IngestArtifacts(ctx context.Context, artifacts []*model.A
UPSERT { algorithm:doc.algorithm, digest:doc.digest }
INSERT { algorithm:doc.algorithm, digest:doc.digest }
UPDATE {} IN artifacts OPTIONS { indexHint: "byArtAndDigest" }
RETURN {
"id": NEW._id,
"algorithm": NEW.algorithm,
"digest": NEW.digest
}`
RETURN { "id": NEW._id }`

sb.WriteString(query)

Expand All @@ -220,34 +216,39 @@ RETURN {
}
defer cursor.Close()

return getArtifacts(ctx, cursor)
createdArtifacts, err := getArtifacts(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get artifact IDs from arango cursor: %w", err)
}

var artifactIDs []string
for _, art := range createdArtifacts {
artifactIDs = append(artifactIDs, art.ID)
}
return artifactIDs, nil
}

func (c *arangoClient) IngestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec) (*model.Artifact, error) {
func (c *arangoClient) IngestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec) (string, error) {
query := `
UPSERT { algorithm:@algorithm, digest:@digest }
INSERT { algorithm:@algorithm, digest:@digest }
UPDATE {} IN artifacts OPTIONS { indexHint: "byArtAndDigest" }
RETURN {
"id": NEW._id,
"algorithm": NEW.algorithm,
"digest": NEW.digest
}`
RETURN { "id": NEW._id }`

cursor, err := executeQueryWithRetry(ctx, c.db, query, getArtifactQueryValues(artifact), "IngestArtifact")
if err != nil {
return nil, fmt.Errorf("failed to ingest artifact: %w", err)
return "", fmt.Errorf("failed to ingest artifact: %w", err)
}
defer cursor.Close()

createdArtifacts, err := getArtifacts(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get artifacts from arango cursor: %w", err)
return "", fmt.Errorf("failed to get artifact IDs from arango cursor: %w", err)
}
if len(createdArtifacts) == 1 {
return createdArtifacts[0], nil
return createdArtifacts[0].ID, nil
} else {
return nil, fmt.Errorf("number of artifacts ingested is greater than one")
return "", fmt.Errorf("number of artifacts ingested is greater than one")
}
}

Expand Down
98 changes: 39 additions & 59 deletions pkg/assembler/backends/arangodb/artifact_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,21 +41,20 @@ func lessArtifact(a, b *model.Artifact) int {
return strings.Compare(a.Digest, b.Digest)
}

func Test_IngestArtifacts(t *testing.T) {
func Test_IngestArtifactIDs(t *testing.T) {
ctx := context.Background()
arangArg := getArangoConfig()
err := deleteDatabase(ctx, arangArg)
arangoArgs := getArangoConfig()
err := deleteDatabase(ctx, arangoArgs)
if err != nil {
t.Fatalf("error deleting arango database: %v", err)
}
c, err := getBackend(ctx, arangArg)
c, err := getBackend(ctx, arangoArgs)
if err != nil {
t.Fatalf("error creating arango backend: %v", err)
}
tests := []struct {
name string
artifactInputs []*model.ArtifactInputSpec
want []*model.Artifact
wantErr bool
}{{
name: "sha256",
Expand All @@ -69,113 +68,94 @@ func Test_IngestArtifacts(t *testing.T) {
Algorithm: "sha512",
Digest: "374ab8f711235830769aa5f0b31ce9b72c5670074b34cb302cdafe3b606233ee92ee01e298e5701f15cc7087714cd9abd7ddb838a6e1206b3642de16d9fc9dd7",
}},
want: []*model.Artifact{{
Algorithm: "sha512",
Digest: "374ab8f711235830769aa5f0b31ce9b72c5670074b34cb302cdafe3b606233ee92ee01e298e5701f15cc7087714cd9abd7ddb838a6e1206b3642de16d9fc9dd7",
}, {
Algorithm: "sha256",
Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf",
}, {
Algorithm: "sha1",
Digest: "7a8f47318e4676dacb0142afa0b83029cd7befd9",
}},
wantErr: false,
}}

ignoreID := cmp.FilterPath(func(p cmp.Path) bool {
return strings.Compare(".ID", p[len(p)-1].String()) == 0
}, cmp.Ignore())
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := c.IngestArtifacts(ctx, tt.artifactInputs)
if (err != nil) != tt.wantErr {
t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("arangoClient.IngestArtifacts() error = %v, wantErr %v", err, tt.wantErr)
return
}
slices.SortFunc(got, lessArtifact)
if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" {
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
if len(got) != len(tt.artifactInputs) {
t.Errorf("Unexpected number of results. Wanted: %d, got %d", len(tt.artifactInputs), len(got))
}
})
}
}

func Test_IngestArtifact(t *testing.T) {
func Test_IngestArtifactID(t *testing.T) {
ctx := context.Background()
arangArg := getArangoConfig()
err := deleteDatabase(ctx, arangArg)
arangoArgs := getArangoConfig()
err := deleteDatabase(ctx, arangoArgs)
if err != nil {
t.Fatalf("error deleting arango database: %v", err)
}
c, err := getBackend(ctx, arangArg)
c, err := getBackend(ctx, arangoArgs)
if err != nil {
t.Fatalf("error creating arango backend: %v", err)
}
tests := []struct {
name string
artifactInput *model.ArtifactInputSpec
want *model.Artifact
wantID bool
wantErr bool
}{{
name: "sha256",
artifactInput: &model.ArtifactInputSpec{
Algorithm: "sha256",
Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf",
},
want: &model.Artifact{
Algorithm: "sha256",
Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf",
},
wantID: true,
wantErr: false,
}, {
name: "sha1",
artifactInput: &model.ArtifactInputSpec{
Algorithm: "sha1",
Digest: "7A8F47318E4676DACB0142AFA0B83029CD7BEFD9",
},
want: &model.Artifact{
Algorithm: "sha1",
Digest: "7a8f47318e4676dacb0142afa0b83029cd7befd9",
},
wantID: true,
wantErr: false,
}, {
name: "sha512",
artifactInput: &model.ArtifactInputSpec{
Algorithm: "sha512",
Digest: "374AB8F711235830769AA5F0B31CE9B72C5670074B34CB302CDAFE3B606233EE92EE01E298E5701F15CC7087714CD9ABD7DDB838A6E1206B3642DE16D9FC9DD7",
},
want: &model.Artifact{
wantID: true,
wantErr: false,
}, {
name: "duplicate sha512",
artifactInput: &model.ArtifactInputSpec{
Algorithm: "sha512",
Digest: "374ab8f711235830769aa5f0b31ce9b72c5670074b34cb302cdafe3b606233ee92ee01e298e5701f15cc7087714cd9abd7ddb838a6e1206b3642de16d9fc9dd7",
Digest: "374AB8F711235830769AA5F0B31CE9B72C5670074B34CB302CDAFE3B606233EE92EE01E298E5701F15CC7087714CD9ABD7DDB838A6E1206B3642DE16D9FC9DD7",
},
wantID: true,
wantErr: false,
}}

ignoreID := cmp.FilterPath(func(p cmp.Path) bool {
return strings.Compare(".ID", p[len(p)-1].String()) == 0
}, cmp.Ignore())
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := c.IngestArtifact(ctx, tt.artifactInput)
if (err != nil) != tt.wantErr {
t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("arangoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
return
}
if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" {
t.Errorf("Unexpected results. (-want +got):\n%s", diff)
if (got != "") != tt.wantID {
t.Errorf("Unexpected number of results")
return
}
})
}
}

func Test_Artifacts(t *testing.T) {
ctx := context.Background()
arangArg := getArangoConfig()
err := deleteDatabase(ctx, arangArg)
arangoArgs := getArangoConfig()
err := deleteDatabase(ctx, arangoArgs)
if err != nil {
t.Fatalf("error deleting arango database: %v", err)
}
c, err := getBackend(ctx, arangArg)
c, err := getBackend(ctx, arangoArgs)
if err != nil {
t.Fatalf("error creating arango backend: %v", err)
}
Expand Down Expand Up @@ -238,17 +218,17 @@ func Test_Artifacts(t *testing.T) {
}, cmp.Ignore())
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ingestedArt, err := c.IngestArtifact(ctx, tt.artifactInput)
ingestedArtID, err := c.IngestArtifact(ctx, tt.artifactInput)
if (err != nil) != tt.wantErr {
t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("arangoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.idInFilter {
tt.artifactSpec.ID = &ingestedArt.ID
tt.artifactSpec.ID = ptrfrom.String(ingestedArtID)
}
got, err := c.Artifacts(ctx, tt.artifactSpec)
if (err != nil) != tt.wantErr {
t.Errorf("demoClient.Artifacts() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("arangoClient.Artifacts() error = %v, wantErr %v", err, tt.wantErr)
return
}
slices.SortFunc(got, lessArtifact)
Expand All @@ -261,12 +241,12 @@ func Test_Artifacts(t *testing.T) {

func Test_buildArtifactResponseByID(t *testing.T) {
ctx := context.Background()
arangArg := getArangoConfig()
err := deleteDatabase(ctx, arangArg)
arangoArgs := getArangoConfig()
err := deleteDatabase(ctx, arangoArgs)
if err != nil {
t.Fatalf("error deleting arango database: %v", err)
}
b, err := getBackend(ctx, arangArg)
b, err := getBackend(ctx, arangoArgs)
if err != nil {
t.Fatalf("error creating arango backend: %v", err)
}
Expand Down Expand Up @@ -329,15 +309,15 @@ func Test_buildArtifactResponseByID(t *testing.T) {
}, cmp.Ignore())
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ingestedArt, err := b.IngestArtifact(ctx, tt.artifactInput)
ingestedArtID, err := b.IngestArtifact(ctx, tt.artifactInput)
if (err != nil) != tt.wantErr {
t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("arangoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.idInFilter {
tt.artifactSpec.ID = &ingestedArt.ID
tt.artifactSpec.ID = ptrfrom.String(ingestedArtID)
}
got, err := b.(*arangoClient).buildArtifactResponseByID(ctx, ingestedArt.ID, tt.artifactSpec)
got, err := b.(*arangoClient).buildArtifactResponseByID(ctx, ingestedArtID, tt.artifactSpec)
if (err != nil) != tt.wantErr {
t.Errorf("arangoClient.buildPackageResponseFromID() error = %v, wantErr %v", err, tt.wantErr)
return
Expand Down
34 changes: 17 additions & 17 deletions pkg/assembler/backends/arangodb/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ func getBuilderQueryValues(builder *model.BuilderInputSpec) map[string]any {
return values
}

func (c *arangoClient) IngestBuilders(ctx context.Context, builders []*model.BuilderInputSpec) ([]*model.Builder, error) {

func (c *arangoClient) IngestBuilders(ctx context.Context, builders []*model.BuilderInputSpec) ([]string, error) {
var listOfValues []map[string]any

for i := range builders {
Expand Down Expand Up @@ -93,15 +92,11 @@ func (c *arangoClient) IngestBuilders(ctx context.Context, builders []*model.Bui
}
}
sb.WriteString("]")

query := `
UPSERT { uri:doc.uri }
INSERT { uri:doc.uri }
UPDATE {} IN builders OPTIONS { indexHint: "byUri" }
RETURN {
"id": NEW._id,
"uri": NEW.uri,
}`
RETURN { "id": NEW._id }`

sb.WriteString(query)

Expand All @@ -111,34 +106,39 @@ RETURN {
}
defer cursor.Close()

return getBuilders(ctx, cursor)
createdBuilders, err := getBuilders(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get builders from arango cursor: %w", err)
}

var builderIDs []string
for _, build := range createdBuilders {
builderIDs = append(builderIDs, build.ID)
}
return builderIDs, nil
}

func (c *arangoClient) IngestBuilder(ctx context.Context, builder *model.BuilderInputSpec) (*model.Builder, error) {
func (c *arangoClient) IngestBuilder(ctx context.Context, builder *model.BuilderInputSpec) (string, error) {
query := `
UPSERT { uri:@uri }
INSERT { uri:@uri }
UPDATE {} IN builders OPTIONS { indexHint: "byUri" }
RETURN {
"id": NEW._id,
"uri": NEW.uri,
}`
RETURN { "id": NEW._id }`

cursor, err := executeQueryWithRetry(ctx, c.db, query, getBuilderQueryValues(builder), "IngestBuilder")
if err != nil {
return nil, fmt.Errorf("failed to ingest builder: %w", err)
return "", fmt.Errorf("failed to ingest builder: %w", err)
}
defer cursor.Close()

createdBuilders, err := getBuilders(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get builders from arango cursor: %w", err)
return "", fmt.Errorf("failed to get builders from arango cursor: %w", err)
}
if len(createdBuilders) == 1 {
return createdBuilders[0], nil
return createdBuilders[0].ID, nil
} else {
return nil, fmt.Errorf("number of builders ingested is greater than one")
return "", fmt.Errorf("number of builders ingested is greater than one")
}
}

Expand Down
Loading

0 comments on commit 32697ae

Please sign in to comment.