feat: add fail-on arg

Added --fail-on argument for snyk test CLI command.
When supplied this argument will check whether any vulns
are upgradable or patchable before failing.
If a vulnerabilities are found be they cannot be upgraded
or patched, a sucessful error code is returned instead.
Test cases added.
Added help text for new arg.

help/help.txt

@@ -63,6 +63,12 @@ Options:
   --remote-repo-url=<string>
                        (monitor command only)
                        Set or override the remote URL for the repository that you would like to monitor.
+  --fail-on=<all|upgradable|patchable>
+                       Only fail when there are vulnerabilities that can be fixed.
+                       All fails when there is at least one vulnerability that can be either upgraded or patched.
+                       Upgradable fails when there is at least one vulnerability that can be upgraded.
+                       Patchable fails when there is at least one vulnerability that can be patched.
+                       If vulnerabilities do not have a fix and this option is being used tests will pass.
 
 Gradle options:
   --sub-project=<string> (alias: --gradle-sub-project)

src/cli/args.ts

@@ -179,6 +179,7 @@ export function args(rawArgv: string[]): Args {
     'gradle-sub-project',
     'skip-unresolved',
     'scan-all-unmanaged',
+    'fail-on',
   ]) {
     if (argv[dashedArg]) {
       const camelCased = dashToCamelCase(dashedArg);

src/cli/commands/test/index.ts

@@ -6,7 +6,7 @@ import * as snyk from '../../../lib';
 import * as config from '../../../lib/config';
 import { isCI } from '../../../lib/is-ci';
 import { apiTokenExists } from '../../../lib/api-token';
-import { SEVERITIES } from '../../../lib/snyk-test/common';
+import { SEVERITIES, FAIL_ON, FailOn } from '../../../lib/snyk-test/common';
 import * as Debug from 'debug';
 import { Options, TestOptions, ShowVulnPaths } from '../../../lib/types';
 import { isLocalFolder } from '../../../lib/detect';
@@ -25,6 +25,7 @@ import {
 } from './formatters/remediation-based-format-issues';
 import * as analytics from '../../../lib/analytics';
 import { isFeatureFlagSupportedForOrg } from '../../../lib/feature-flags';
+import { FailOnError } from '../../../lib/errors/fail-on-error.ts';
 
 const debug = Debug('snyk');
 const SEPARATOR = '\n-------------------------------------------------------\n';
@@ -66,6 +67,11 @@ async function test(...args: MethodArgs): Promise<string> {
     return Promise.reject(new Error('INVALID_SEVERITY_THRESHOLD'));
   }
 
+  if (options.failOn && !validateFailOn(options.failOn)) {
+    const error = new FailOnError();
+    return Promise.reject(chalk.red.bold(error.message));
+  }
+
   apiTokenExists();
 
   // Promise waterfall to test all other paths sequentially
@@ -151,20 +157,29 @@ async function test(...args: MethodArgs): Promise<string> {
 
     // backwards compat - strip array IFF only one result
     const dataToSend = results.length === 1 ? results[0] : results;
-    const stringifiedError = JSON.stringify(dataToSend, null, 2);
+    const stringifiedData = JSON.stringify(dataToSend, null, 2);
 
     if (results.every((res) => res.ok)) {
-      return stringifiedError;
+      return stringifiedData;
     }
-    const err = new Error(stringifiedError) as any;
+
+    const err = new Error(stringifiedData) as any;
+
     if (foundVulnerabilities) {
+      if (options.failOn) {
+        const fail = shouldFail(vulnerableResults, options.failOn);
+        if (!fail) {
+          // return here to prevent failure
+          return stringifiedData;
+        }
+      }
       err.code = 'VULNS';
       const dataToSendNoVulns = dataToSend;
       delete dataToSendNoVulns.vulnerabilities;
       err.jsonNoVulns = dataToSendNoVulns;
     }
 
-    err.json = stringifiedError;
+    err.json = stringifiedData;
     throw err;
   }
 
@@ -215,6 +230,15 @@ async function test(...args: MethodArgs): Promise<string> {
   }
 
   if (foundVulnerabilities) {
+    if (options.failOn) {
+      const fail = shouldFail(vulnerableResults, options.failOn);
+      if (!fail) {
+        // return here to prevent throwing failure
+        response += chalk.bold.green(summaryMessage);
+        return response;
+      }
+    }
+
     response += chalk.bold.red(summaryMessage);
     const error = new Error(response) as any;
     // take the code of the first problem to go through error
@@ -230,6 +254,63 @@ async function test(...args: MethodArgs): Promise<string> {
   return response;
 }
 
+function shouldFail(vulnerableResults: any[], failOn: FailOn) {
+  // find reasons not to fail
+  if (failOn === 'all') {
+    return hasFixes(vulnerableResults);
+  }
+  if (failOn === 'upgradable') {
+    return hasUpgrades(vulnerableResults);
+  }
+  if (failOn === 'patchable') {
+    return hasPatches(vulnerableResults);
+  }
+  // should fail by default when there are vulnerable results
+  return vulnerableResults.length > 0;
+}
+
+function hasFix(vuln: any) {
+  const { isUpgradable, isPinnable, isPatchable } = vuln;
+  return isUpgradable || isPinnable || isPatchable;
+}
+
+function hasUpgrade(vuln: any) {
+  const { isUpgradable, isPinnable } = vuln;
+  return isUpgradable || isPinnable;
+}
+
+function hasPatch(vuln: any) {
+  const { isPatchable } = vuln;
+  return isPatchable;
+}
+
+function isTestResultFixable(testResult: any): boolean {
+  const { vulnerabilities } = testResult;
+  return vulnerabilities.some(hasFix);
+}
+
+function hasFixes(testResults: any[]): boolean {
+  return testResults.some(isTestResultFixable);
+}
+
+function isTestResultUpgradable(testResult: any): boolean {
+  const { vulnerabilities } = testResult;
+  return vulnerabilities.some(hasUpgrade);
+}
+
+function hasUpgrades(testResults: any[]): boolean {
+  return testResults.some(isTestResultUpgradable);
+}
+
+function isTestResultPatchable(testResult: any): boolean {
+  const { vulnerabilities } = testResult;
+  return vulnerabilities.some(hasPatch);
+}
+
+function hasPatches(testResults: any[]): boolean {
+  return testResults.some(isTestResultPatchable);
+}
+
 function summariseVulnerableResults(vulnerableResults, options: TestOptions) {
   const vulnsLength = vulnerableResults.length;
   if (vulnsLength) {
@@ -575,6 +656,10 @@ function validateSeverityThreshold(severityThreshold) {
   return SEVERITIES.map((s) => s.verboseName).indexOf(severityThreshold) > -1;
 }
 
+function validateFailOn(arg: FailOn) {
+  return Object.keys(FAIL_ON).includes(arg);
+}
+
 // This is all a copy from Registry snapshots/index
 function isVulnFixable(vuln) {
   return vuln.isUpgradable || vuln.isPatchable;

src/lib/errors/fail-on-error.ts.ts

@@ -0,0 +1,12 @@
+import { CustomError } from './custom-error';
+import { FAIL_ON } from '../snyk-test/common';
+
+export class FailOnError extends CustomError {
+  private static ERROR_MESSAGE =
+    'Invalid fail on argument, please use one of: ' +
+    Object.keys(FAIL_ON).join(' | ');
+
+  constructor() {
+    super(FailOnError.ERROR_MESSAGE);
+  }
+}

src/lib/snyk-test/common.ts

@@ -42,3 +42,11 @@ export const SEVERITIES: Array<{
     value: 3,
   },
 ];
+
+export enum FAIL_ON {
+  all = 'all',
+  upgradable = 'upgradable',
+  patchable = 'patchable',
+}
+
+export type FailOn = 'all' | 'upgradable' | 'patchable';

src/lib/snyk-test/run-test.ts

@@ -207,7 +207,6 @@ function sendTestPayload(
       }
 
       body.filesystemPolicy = filesystemPolicy;
-
       resolve(body);
     });
   });

src/lib/types.ts

@@ -1,6 +1,7 @@
 import { SupportedPackageManagers } from './package-managers';
 import { legacyCommon as legacyApi } from '@snyk/cli-interface';
 import { SEVERITY } from './snyk-test/legacy';
+import { FailOn } from './snyk-test/common';
 
 export interface PluginMetadata {
   name: string;
@@ -29,6 +30,7 @@ export interface TestOptions {
   'prune-repeated-subdependencies'?: boolean;
   showVulnPaths: ShowVulnPaths;
   pinningSupported?: boolean;
+  failOn?: FailOn;
 }
 export interface ProtectOptions {
   loose: boolean;

test/acceptance/cli-fail-on-pinnable.test.ts

@@ -0,0 +1,129 @@
+import * as tap from 'tap';
+import * as cli from '../../src/cli/commands';
+import { fakeServer } from './fake-server';
+import * as version from '../../src/lib/version';
+import * as sinon from 'sinon';
+import * as snyk from '../../src/lib';
+import { getWorkspaceJSON, chdirWorkspaces } from './workspace-helper';
+
+const { test, only } = tap;
+(tap as any).runOnly = false; // <- for debug. set to true, and replace a test to only(..)
+
+const port = (process.env.PORT = process.env.SNYK_PORT = '12345');
+process.env.SNYK_API = 'http://localhost:' + port + '/api/v1';
+process.env.SNYK_HOST = 'http://localhost:' + port;
+process.env.LOG_LEVEL = '0';
+const apiKey = '123456789';
+let oldkey;
+let oldendpoint;
+let versionNumber;
+const server = fakeServer(process.env.SNYK_API, apiKey);
+const before = tap.runOnly ? only : test;
+const after = tap.runOnly ? only : test;
+
+const pinnableVulnsResult = getWorkspaceJSON(
+  'fail-on',
+  'pinnable',
+  'vulns-result.json',
+);
+
+// snyk test stub responses
+const pinnableVulns = getWorkspaceJSON('fail-on', 'pinnable', 'vulns.json');
+
+// @later: remove this config stuff.
+// Was copied straight from ../src/cli-server.js
+before('setup', async (t) => {
+  versionNumber = await version();
+
+  t.plan(3);
+  let key = await cli.config('get', 'api');
+  oldkey = key;
+  t.pass('existing user config captured');
+
+  key = await cli.config('get', 'endpoint');
+  oldendpoint = key;
+  t.pass('existing user endpoint captured');
+
+  await new Promise((resolve) => {
+    server.listen(port, resolve);
+  });
+  t.pass('started demo server');
+  t.end();
+});
+
+// @later: remove this config stuff.
+// Was copied straight from ../src/cli-server.js
+before('prime config', async (t) => {
+  await cli.config('set', 'api=' + apiKey);
+  t.pass('api token set');
+  await cli.config('unset', 'endpoint');
+  t.pass('endpoint removed');
+  t.end();
+});
+
+test('test vulnerable project with pinnable and --fail-on=upgradable', async (t) => {
+  // mocking test results here as CI tooling does not have python installed
+  const snykTestStub = sinon.stub(snyk, 'test').returns(pinnableVulns);
+  try {
+    server.setNextResponse(pinnableVulnsResult);
+    chdirWorkspaces('fail-on');
+    await cli.test('pinnable', {
+      failOn: 'upgradable',
+    });
+    t.fail('expected test to throw exception');
+  } catch (err) {
+    t.equal(err.code, 'VULNS', 'should throw exception');
+  } finally {
+    snykTestStub.restore();
+  }
+});
+
+test('test vulnerable project with pinnable and --fail-on=upgradable --json', async (t) => {
+  // mocking test results here as CI tooling does not have python installed
+  const snykTestStub = sinon.stub(snyk, 'test').returns(pinnableVulns);
+  try {
+    server.setNextResponse(pinnableVulnsResult);
+    chdirWorkspaces('fail-on');
+    await cli.test('pinnable', {
+      failOn: 'upgradable',
+      json: true,
+    });
+    t.fail('expected test to throw exception');
+  } catch (err) {
+    t.equal(err.code, 'VULNS', 'should throw exception');
+  } finally {
+    snykTestStub.restore();
+  }
+});
+
+// @later: try and remove this config stuff
+// Was copied straight from ../src/cli-server.js
+after('teardown', async (t) => {
+  t.plan(4);
+
+  delete process.env.SNYK_API;
+  delete process.env.SNYK_HOST;
+  delete process.env.SNYK_PORT;
+  t.notOk(process.env.SNYK_PORT, 'fake env values cleared');
+
+  await new Promise((resolve) => {
+    server.close(resolve);
+  });
+  t.pass('server shutdown');
+  let key = 'set';
+  let value = 'api=' + oldkey;
+  if (!oldkey) {
+    key = 'unset';
+    value = 'api';
+  }
+  await cli.config(key, value);
+  t.pass('user config restored');
+  if (oldendpoint) {
+    await cli.config('endpoint', oldendpoint);
+    t.pass('user endpoint restored');
+    t.end();
+  } else {
+    t.pass('no endpoint');
+    t.end();
+  }
+});