Generate and validate SCW artifacts (#272)

Author: adamrdavid

.github/workflows/validate_artifacts.yml

@@ -0,0 +1,40 @@
+name: Validate Artifacts
+
+on:
+  push:
+    branches: 
+      - master
+  release:
+    types:
+      - created
+      - edited
+      - unpublished
+
+jobs:
+  validate_scw_artifact:
+    name: Validate SCW Artifact
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r lib/requirements.txt
+      - name: Install requests library
+        run: |
+          pip install requests
+      - name: Create artifact json file
+        run: |
+          python3 -B lib/generate_artifacts.py
+      - name: Upload artifact
+        uses: actions/upload-artifact@v1
+        with:
+          name: Secure Code Warrior Links
+          path: scw_links.json
+      - name: Validate links
+        run: |
+          python3 -B lib/validate_artifacts.py

.github/workflows/main.yml renamed to .github/workflows/validate_vrt.yml

@@ -16,14 +16,14 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r tests/requirements.txt
+        pip install -r lib/requirements.txt
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 ./lib --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 ./lib --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with unittest
       run: |
-        python3 -B validate_vrt.py
+        python3 -B lib/validate_vrt.py

Dockerfile

@@ -3,4 +3,4 @@ FROM python:3.6
 RUN pip install jsonschema GitPython semantic_version
 
 WORKDIR /tmp/vrt
-CMD [ "python3", "-B" , "./validate_vrt.py" ]
+CMD [ "python3", "-B" , "./lib/validate_vrt.py" ]

README.md

@@ -166,6 +166,9 @@ Each mapping should be setup in the following structure:
 - [CWE](mappings/cwe/cwe.json)
 - [Remediation Advice](mappings/remediation_advice/remediation_advice.json)
 
+#### Remediation Training
+- [Secure Code Warriors](remediation_training/)
+
 ## Supported Libraries
 - [Ruby](https://github.com/bugcrowd/vrt-ruby)
 

lib/artifacts/__init__.py

@@ -0,0 +1,47 @@
+import json
+import requests
+import utils.utils
+
+BASE_SCW_URL = 'https://integration-api.securecodewarrior.com/api/v1/trial?id=bugcrowd&mappingList=vrt&mappingKey='
+OUTPUT_FILENAME = 'scw_links.json'
+
+
+def scw_url(vrt_id):
+    return f'{BASE_SCW_URL}{vrt_id.replace(".", ":")}'
+
+
+def scw_mapping(vrt_id):
+    path = scw_url(vrt_id)
+    print('Fetching...')
+    response = requests.get(path)
+    if response.status_code == 200:
+        print(f'Exists: {path}')
+        return path + '&redirect=true'
+    else:
+        print(f'Not Found: {path}')
+        return None
+
+
+def join_vrt_id(parent_id, child_id):
+    return '.'.join([parent_id, child_id]) if parent_id is not None else child_id
+
+
+def generate_urls(vrt, content, parent_id=None):
+    for node in vrt:
+        vrt_id = join_vrt_id(parent_id, node['id'])
+        content[vrt_id] = scw_mapping(vrt_id)
+        if 'children' in node:
+            content.update(
+                generate_urls(
+                    node['children'],
+                    {},
+                    vrt_id
+                )
+            )
+
+    return content
+
+
+def write_artifact_file(mapping):
+    with open(OUTPUT_FILENAME, 'w') as outfile:
+        json.dump(mapping, outfile, indent=2, sort_keys=False)

lib/artifacts/scw_artifact.py

@@ -0,0 +1,8 @@
+from utils import utils
+from artifacts import scw_artifact
+
+url_mapping = {}
+current_vrt = utils.get_json(utils.VRT_FILENAME)
+scw_artifact.write_artifact_file(
+  scw_artifact.generate_urls(current_vrt['content'], url_mapping)
+)

lib/generate_artifacts.py

@@ -1,4 +1,4 @@
-import utils
+from utils import utils
 import unittest
 from semantic_version import Version