From 21533256908795081d43667ee5ae75704be3101f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 1 Nov 2024 20:41:26 +0100 Subject: [PATCH 1/6] Save scan record async Signed-off-by: Paolo Di Tommaso --- .../persistence/impl/SurrealClient.groovy | 6 ++++ .../impl/SurrealPersistenceService.groovy | 28 +++++++++++++------ .../impl/SurrealPersistenceServiceTest.groovy | 6 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index badd7c4aa..2901ec22f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -54,9 +54,15 @@ interface SurrealClient { @Post("/sql") Flux> sqlAsync(@Header String authorization, @Body String body) + @Post("/sql") + Flux>> sqlAsyncMany(@Header String authorization, @Body String body) + @Post("/sql") Map sqlAsMap(@Header String authorization, @Body String body) + @Post("/sql") + List> sqlAsList(@Header String authorization, @Body String body) + @Post("/sql") String sqlAsString(@Header String authorization, @Body String body) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 74079f1b7..53c491131 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -256,25 +256,35 @@ class SurrealPersistenceService implements PersistenceService { void saveScanRecord(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() + List ids = new ArrayList<>(100) + String statement = '' // save all vulnerabilities for( ScanVulnerability it : vulnerabilities ) { - surrealDb.insertScanVulnerability(authorization, it) + statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n" + ids << "wave_scan_vuln:⟨$it.id⟩".toString() } - // compose the list of ids - final ids = vulnerabilities - .collect(it-> "wave_scan_vuln:⟨$it.id⟩".toString()) - - // scan object final copy = scanRecord.clone() copy.vulnerabilities = List.of() final json = JacksonHelper.toJson(copy) // create the scan record - final statement = "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)}".toString() - final result = surrealDb.sqlAsMap(authorization, statement) - log.trace "Scan update result=$result" + statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() + + surrealDb + .sqlAsyncMany(getAuthorization(), statement) + .subscribe({result -> + log.trace "Scan update result=$result" + }, + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error updating scan record => ${msg}\n", error) + }) + } protected String patchScanVulnerabilities(String json, List ids) { diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index ea27f24b7..43227ad5c 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -324,6 +324,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: persistence.saveScanRecord(scan) + sleep 200 then: def result = persistence.loadScanRecord(SCAN_ID) and: @@ -343,6 +344,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe and: // should save the same CVE into another build persistence.saveScanRecord(scanRecord2) + sleep 200 then: def result2 = persistence.loadScanRecord(SCAN_ID2) and: @@ -372,6 +374,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe when: persistence.saveScanRecord(scan) + sleep 200 then: persistence.existsScanRecord(SCAN_ID) } @@ -566,7 +569,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe persistence.saveScanRecord(scan2) persistence.saveScanRecord(scan3) persistence.saveScanRecord(scan4) - + and: + sleep 200 then: persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1] and: From 73d84c91b64430e53ac7495e5d6456d69249d160 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 1 Nov 2024 20:44:49 +0100 Subject: [PATCH 2/6] Add comment [ci skip] Signed-off-by: Paolo Di Tommaso --- .../service/persistence/impl/SurrealPersistenceService.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 53c491131..a54459ec1 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -269,9 +269,9 @@ class SurrealPersistenceService implements PersistenceService { copy.vulnerabilities = List.of() final json = JacksonHelper.toJson(copy) - // create the scan record + // add the wave_scan record statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() - + // store the statement using an async operation surrealDb .sqlAsyncMany(getAuthorization(), statement) .subscribe({result -> From 2c99ea412486cbd02b8c382693fc96596a521380 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 08:34:49 +0100 Subject: [PATCH 3/6] Minor change Signed-off-by: Paolo Di Tommaso --- .../impl/SurrealPersistenceService.groovy | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index a54459ec1..64ad6f7c8 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -256,7 +256,9 @@ class SurrealPersistenceService implements PersistenceService { void saveScanRecord(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() - List ids = new ArrayList<>(100) + // create a multi-command surreal sql statement to insert all vulnerabilities + // and the scan record in a single operation + List ids = new ArrayList<>(101) String statement = '' // save all vulnerabilities for( ScanVulnerability it : vulnerabilities ) { @@ -275,16 +277,15 @@ class SurrealPersistenceService implements PersistenceService { surrealDb .sqlAsyncMany(getAuthorization(), statement) .subscribe({result -> - log.trace "Scan update result=$result" + log.trace "Scan record save result=$result" }, - {error-> - def msg = error.message - if( error instanceof HttpClientResponseException ){ - msg += ":\n $error.response.body" - } - log.error("Error updating scan record => ${msg}\n", error) - }) - + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving scan record => ${msg}\n", error) + }) } protected String patchScanVulnerabilities(String json, List ids) { From 5f2c7ae5e2c6f438897d71be36604c9e3a7a5fa0 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 08:35:45 +0100 Subject: [PATCH 4/6] Bump version 1.13.11-B1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f3352b3fe..bb3be891d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.10 +1.13.11-B1 From 1f1438c68cf15b93f0c2f11b9b14f5f32508cdec Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 09:04:31 +0100 Subject: [PATCH 5/6] Refactor Signed-off-by: Paolo Di Tommaso --- .../wave/configuration/ScanConfig.groovy | 1 + .../scan/ContainerScanServiceImpl.groovy | 2 +- .../service/scan/TrivyResultProcessor.groovy | 24 ++++++++---- .../scan/TrivyResultProcessorTest.groovy | 38 +++++++++++-------- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index a3232f300..baef40d75 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -84,6 +84,7 @@ class ScanConfig { @Value('${wave.scan.environment}') List environment + @Nullable @Value('${wave.scan.vulnerability.limit:100}') Integer vulnerabilityLimit diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 1571bb50d..73305463e 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -270,7 +270,7 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler parse(Path scanFile) { - return parse(scanFile.getText()) - } - - static List parse(Path scanFile, int maxEntries) { - final result = parse(scanFile) - return filter(result, maxEntries) + /** + * Parse a Trivy vulnerabilities JSON file and return a list of {@link ScanVulnerability} entries + * + * @param scanFile + * The {@link Path} of the Trivy JSON file to be scanned + * @param maxEntries + * The max number of vulnerabilities that should be returned giving precedence to the + * most severe vulnerabilities e.g. one critical and one medium issues are found and + * 1 is specified as {@code maxEntries} only the critical issues is returned. + * @return + * The list of {@link ScanVulnerability} entries as parsed in from the JSON file. + */ + static List parseFile(Path scanFile, Integer maxEntries=null) { + final result = parseJson(scanFile.getText()) + return maxEntries>0 ? filter(result, maxEntries) : result } @CompileDynamic - static List parse(String scanJson) { + static List parseJson(String scanJson) { final slurper = new JsonSlurper() try{ final jsonMap = slurper.parseText(scanJson) as Map diff --git a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy index 8bb0186b6..af9b4b549 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy @@ -18,11 +18,11 @@ package io.seqera.wave.service.scan - import spock.lang.Specification -import io.seqera.wave.exception.ScanRuntimeException +import java.nio.file.Files +import io.seqera.wave.exception.ScanRuntimeException /** * * @author Munish Chouhan @@ -91,7 +91,7 @@ class TrivyResultProcessorTest extends Specification { """ when: - def result = TrivyResultProcessor.parse(trivyDockerResulJson) + def result = TrivyResultProcessor.parseJson(trivyDockerResulJson) then: def vulnerability = result[0] @@ -107,7 +107,10 @@ class TrivyResultProcessorTest extends Specification { def "should return a sorted map of vulnerabilities"() { given: - def trivyDockerResulJson = """ + def folder = Files.createTempDirectory('test') + def scan = folder.resolve('scan.json') + and: + scan.text = """ { "Results": [ { "Target": "sample-application", @@ -170,19 +173,22 @@ class TrivyResultProcessorTest extends Specification { }""".stripIndent() when: - def result = TrivyResultProcessor.parse(trivyDockerResulJson) - result = TrivyResultProcessor.filter(result, 4) + def topIssues = TrivyResultProcessor.parseFile(scan, 2) then: - result.size() == 4 - result[0].severity == "CRITICAL" - result[0].id == "CVE-2023-0005" - result[1].severity == "HIGH" - result[1].id == "CVE-2023-0003" - result[2].severity == "HIGH" - result[2].id == "CVE-2023-0004" - result[3].severity == "MEDIUM" - result[3].id == "CVE-2023-0002" + topIssues.size() == 2 + topIssues[0].severity == "CRITICAL" + topIssues[0].id == "CVE-2023-0005" + topIssues[1].severity == "HIGH" + topIssues[1].id == "CVE-2023-0003" + + when: + def allIssues = TrivyResultProcessor.parseFile(scan) + then: + allIssues.size() == 5 + + cleanup: + folder?.deleteDir() } def 'should not fail with empty list' () { @@ -192,7 +198,7 @@ class TrivyResultProcessorTest extends Specification { def "process should throw exception if json is not correct"() { when: - TrivyResultProcessor.parse("invalid json") + TrivyResultProcessor.parseJson("invalid json") then: thrown ScanRuntimeException } From 960d166a8e3db57a8be4eeb34c775cdf23f60914 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 2 Nov 2024 09:12:29 +0100 Subject: [PATCH 6/6] Update VERSION [ci skip] --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bb3be891d..f3352b3fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.11-B1 +1.13.10