From db730fee8515a50dac40a591240ed27be1f2fe0b Mon Sep 17 00:00:00 2001 From: Aaron Unterberger Date: Tue, 13 Jul 2021 15:43:25 -0400 Subject: [PATCH] Dome9 and Aqua commit Dome9 and Aqua parsers created and committed --- .DS_Store | Bin 0 -> 6148 bytes aqua/.DS_Store | Bin 0 -> 6148 bytes aqua/.idea/.gitignore | 3 + aqua/.idea/aqua_csv_2nucleus.iml | 8 + .../inspectionProfiles/profiles_settings.xml | 6 + aqua/.idea/misc.xml | 4 + aqua/.idea/modules.xml | 8 + aqua/README.md | 25 +++ aqua/aqua2nucleus_csv.py | 179 ++++++++++++++++++ aqua/envrionment.yml | 9 + aqua/files/.DS_Store | Bin 0 -> 6148 bytes aqua/files/aqua_findings.csv | 1 + dome9/.DS_Store | Bin 0 -> 6148 bytes dome9/.idea/.gitignore | 3 + dome9/.idea/aqua_csv_2nucleus.iml | 8 + .../inspectionProfiles/profiles_settings.xml | 6 + dome9/.idea/misc.xml | 4 + dome9/.idea/modules.xml | 8 + dome9/README.md | 25 +++ dome9/dome9_2nucleus_csv.py | 140 ++++++++++++++ dome9/envrionment.yml | 9 + dome9/files/.DS_Store | Bin 0 -> 6148 bytes dome9/files/dome9_findings.csv | 1 + 23 files changed, 447 insertions(+) create mode 100644 .DS_Store create mode 100644 aqua/.DS_Store create mode 100644 aqua/.idea/.gitignore create mode 100644 aqua/.idea/aqua_csv_2nucleus.iml create mode 100644 aqua/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 aqua/.idea/misc.xml create mode 100644 aqua/.idea/modules.xml create mode 100644 aqua/README.md create mode 100644 aqua/aqua2nucleus_csv.py create mode 100644 aqua/envrionment.yml create mode 100644 aqua/files/.DS_Store create mode 100644 aqua/files/aqua_findings.csv create mode 100644 dome9/.DS_Store create mode 100644 dome9/.idea/.gitignore create mode 100644 dome9/.idea/aqua_csv_2nucleus.iml create mode 100644 dome9/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 dome9/.idea/misc.xml create mode 100644 dome9/.idea/modules.xml create mode 100644 dome9/README.md create mode 100644 dome9/dome9_2nucleus_csv.py create mode 100644 dome9/envrionment.yml create mode 100644 dome9/files/.DS_Store create mode 100644 dome9/files/dome9_findings.csv diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2a6b6199ab6781b50301bf221f28debec7c008d4 GIT binary patch literal 6148 zcmeH~zi$&U6vv<2YjDD^3`lgmrCWz2MMW8qj&L0qija;U1EBVfi<*<}&LO!Yf;8y< z1lIB=@NcldR>Z)B#K6qozH!KE`F5eR(otMMtLDHY>HLu(fc@ZXge(=L^-ZEeqHb$)fDedG3i z_TuIAaQ5m*5oLG)NxQ7|r@X)3WH%wn9d|w}4{QcQCJJ%ov0U7k6d)AO|Bj&A6_pgx~OEjU^j=x@71Ylyn z|N6^rsjwUInfL|%^d2Vq+DG|yjrbGrae8Z#fZj6vyyamDSOS*72?BIJxR^*it(8)F zbYLc*0H_W&OT#j~B`C(#>S?W%q6dwsP*fGl^b>=raBO#VKTm6=RD}c6j}NA2X8MJ~ z?H#nD~s uV%o<S5T30SQoPig$9)2!-~+5RUPY+A*BWibgcQVkp28RM5qu26XYiYyF?O34 zZz4Jav)^WZGUUsW$q?d0Ee4%_=&1e!F-%-Ha%{iC8VyPsj}EsDFq0CF(yW z=Vuz0sdv4;Kf8K;|E%BUSM^u@?GZ`+=_S`UF$RnQW8l9rfS%2g9SGWL3>X8(K+6Ds zA3T&XC^mxe(}5+90Kh)XQLyJ(f^)oLP;3O@fjCJ8N~+Tl!$~^qUgLsdBPi+Qbog*u z+3AGhVs+f#$8d5%&{ku>7)Tk|lglCZ|MT7Rf0|@h#(**Krx + + + + + + + \ No newline at end of file diff --git a/aqua/.idea/inspectionProfiles/profiles_settings.xml b/aqua/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/aqua/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/aqua/.idea/misc.xml b/aqua/.idea/misc.xml new file mode 100644 index 0000000..0b8b24f --- /dev/null +++ b/aqua/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/aqua/.idea/modules.xml b/aqua/.idea/modules.xml new file mode 100644 index 0000000..e397bde --- /dev/null +++ b/aqua/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/aqua/README.md b/aqua/README.md new file mode 100644 index 0000000..2877db8 --- /dev/null +++ b/aqua/README.md @@ -0,0 +1,25 @@ +This script is a command line tool for easily converting Aqua csv output into a Nucleus format for easy upload. The script iterates through the Aqua file, normalizing it into the Nucleus Custom Schema and then writes it to CSV output. The script can also optionally POST the file to your designated Nucleus project. + +Install: + +This project uses Conda to manage the Python virtual environment. + +Conda can be [downloaded here](https://docs.conda.io/projects/conda/en/latest/user-guide/install/download.html) + +The documentation for managing the environment [can be found here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) + +The process for creating the environment is: +1. `conda env create -f environment.yml` with the environment.yml reference to the environment yaml in the project. +2. `conda activate nucleus-plugin` to activate your virtual environment +3. To deactivate the environment, execute `conda deactivate` + +Usage: +- -i for the input Aqua file path +- -o for the output file path for the output csv + +**Example:** `python3 aqua2nucleus_csv.py -i files/aqua_findings.csv -o aqua2nucleus_out.csv -# 11000176` + +Optional: +- - -# Project ID of the Nucleus project to which you to POST the output file + +**Note**: If you want to post the file to Nucleus you will need to get an API key from the Nucleus console and update the NUCLEUS_ROOT_URL in the python script. diff --git a/aqua/aqua2nucleus_csv.py b/aqua/aqua2nucleus_csv.py new file mode 100644 index 0000000..10bc3bd --- /dev/null +++ b/aqua/aqua2nucleus_csv.py @@ -0,0 +1,179 @@ +# Used for writing to csv +import csv +# Used for arguments +import argparse +# Used to post the file to Nucleus +import requests +from datetime import datetime +import re + +# Enter in the root URL of your Nucleus instance WITHOUT the trailing slash. +# Example https://example.nucleussec.com +NUCLEUS_ROOT_URL = "{Enter root URL of your Nucleus Instance here}" +# Generate an API key through the Nucleus UI +API_KEY = "{Enter your API key from Nucleus here}" + + +def customParser(inputPath, outputPath): + with open(inputPath, 'r', newline='', encoding='utf8') as input_file: + findings = csv.reader(input_file) + + # Create the csv file for writing + with open(outputPath, 'w', newline='') as output_file: + + # write headers on output file + csvwriter = csv.writer(output_file, delimiter=',') + csvwriter.writerow(['nucleus_import_version', 'host_name', 'scan_date', 'scan_type', 'scan_tool',\ + 'finding_number', 'finding_name', 'finding_severity', 'finding_type', 'finding_cve',\ + 'finding_description', 'finding_exploitable', 'finding_path', 'finding_recommendation',\ + 'finding_references', 'finding_result', 'asset_info']) + + # skip first line of csv headers + next(findings) + + # iterate through each row in findings and map to nucleus fields + for finding in findings: + csv_line = [] + host_name = finding[1] + scan_date = datetime.strptime(finding[42], '%m/%d/%y') + #find_num = finding[7] + "-" + finding[2] + find_num = finding[7] + find_name = finding[7] + if (finding[39] != "negligible"): + find_sev = finding[39] + else: + find_sev = "informational" + find_cve = finding[7] + find_descr = finding[27] + find_path = finding[4] + if (finding[34] == "patch_available"): + # provide patch information + find_rec = finding[34] + "\n" + "Patch to lastest version: " + finding[25] + else: + # no patch available + find_rec = finding[34] + # os and version mapping + if len(finding[3]) > 1: + os_idx = re.search(r"\d", finding[3]).span()[0] # get index of version in OS string + os_name = finding[3][:(os_idx-1)] + os_version = finding[3][os_idx:] + else: + os_name = "" + os_version = "" + # invoke reference builder + find_refs = create_refs(finding) + # generate asset info + asset_info = "" + if (finding[0]): asset_info = asset_info + "aqua.registry:" + finding[0] + sha = finding[2] + if (finding[2]): asset_info = asset_info + ";aqua.image_digest:" + sha.replace("sha256:", "") + # create csv line array and write to output file + csv_line = ['1', host_name, scan_date, 'Container Image', 'Aqua', find_num, find_name, find_sev, 'Vuln', \ + find_cve, find_descr, "True", find_path, find_rec, find_refs, "Failed", asset_info] + csvwriter.writerow(csv_line) + + output_file.close() + return output_file + +def create_refs(finding): + + # populate ref params + cvss_v3_score = finding[21] + cvss_v3_vec = finding[22] + cvss_v2_score = finding[18] + cvss_v2_vec = finding[19] + publish_date = finding[8] + exploit_type = finding[45] + exploit_avail = finding[43] + temporal_vector = finding[44] + + # create refs string + refs = "" + if (cvss_v3_score): refs = refs + "CVSS V3 Score: " + cvss_v3_score + if (cvss_v3_vec): refs = refs + ",CVSS V3 Vector: " + cvss_v3_vec + if (cvss_v2_score): refs = refs + ",CVSS V2 Score: " + cvss_v2_score + if (cvss_v2_vec): refs = refs + ",CVSS V2 Vector: " + cvss_v2_vec + if (publish_date): refs = refs + ",Publish Date: " + publish_date + if (exploit_type): refs = refs + ",Exploit Type: " + exploit_type + if (exploit_avail): refs = refs + ",Exploit Available: " + exploit_avail + if (temporal_vector): refs = refs + ",Temporal Vector: " + temporal_vector + + return refs + +def get_args(): + parser = argparse.ArgumentParser( + description="For parsing whitesource files to be uploaded into Nucleus. If project ID is specified, will post the Nucleus supported file to Nucleus project.") + + # List arguments. Should only include input file and output file + parser.add_argument('-i', '--inputfile', dest='inputFile', help="Path to whitesource xml file to parse", + required=True) + parser.add_argument('-o', '--outputfile', dest='outputFile', help="Path to csv file output", required=True) + parser.add_argument('-#', '--project_id', dest="project_id", + help="This is the project ID of the Nucleus project to which you want to post. If not specified, this script will only parse the whitesource file for manual upload.") + + # Define the arguments globally for ease of use + global args + + args = parser.parse_args() + + return args + + +# Send the file to Nucleus +def post_to_nucleus(outputfile): + # Enter the ID of the project which you wish to post to here + PROJECT_ID = args.project_id + + # open the file to send + with open(outputfile.name, 'rb') as f: + + # Get the final Nucleus URL to post to + nucleus_url = str(NUCLEUS_ROOT_URL + '/nucleus/api/projects/' + PROJECT_ID + '/scans') + + print("Posted to URL:", nucleus_url) + + # Send file with proper header. Keep note of the project ID you need to send + file_upload = requests.post(nucleus_url, files={outputfile.name: f}, headers={'x-apikey': API_KEY}) + + # Print the response from the server + print(file_upload.content) + + if file_upload.status_code == 404: + + print( + "You probably entered the wrong url. Check to make sure the last slash '/' has been removed from the NUCLEUS_ROOT_URL") + + else: + + pass + + +if __name__ == "__main__": + + # Get the arguments + arguments = get_args() + + # Get the input file to parse + inputPath = arguments.inputFile + + # Get the output file to save to + outputPath = arguments.outputFile + + # Print to cli so user has feedback + print("File received, parsing contents") + + # Start the parsing and csv writing + outputfile = customParser(inputPath, outputPath) + + # If a project ID was specified, send the file to Nucleus + if arguments.project_id: + + print("Parsing complete. Sending to specified Nucleus project") + + # Send the newly created csv file to Nucleus if project id was specified + post_to_nucleus(outputfile) + + # If no project ID was specified, just parse file to Nucleus format for manual file upload + else: + + print("Parsing complete. Success!") diff --git a/aqua/envrionment.yml b/aqua/envrionment.yml new file mode 100644 index 0000000..c3a739f --- /dev/null +++ b/aqua/envrionment.yml @@ -0,0 +1,9 @@ +name: nucleus-plugin + +dependencies: + - python=3.9 + - pip + - pip: + - requests + - numpy + - pandas diff --git a/aqua/files/.DS_Store b/aqua/files/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5c1abe098c3c7acc1f22aac8008c6d57252002a2 GIT binary patch literal 6148 zcmeHKO-}+r41I$fFkbNHaesk`_{D8Q@Me^RXyOU^R>OxbEQz=M>ArRbl%RjjEinaE<^;K_Gn>%Crr?zHQ5FRGzRnw9Sl( z)+Fd?bf~t32D^3f$cltL8njb9bF9m3>G!cVT?yBC%uXrCN6bn!N8UQ|B>$)Pa~(bsoN?9)&T)z=PUs&oy3`8&3;k6Y^=lD!5^8VB;5NJr zwnk&X82C2~aL*Pg?0Gb63>X8(z={F+K15W(IAG>cKOHoB1R&-aR^hdrC8Q<_7zfNe za)jc1Dbbg@aKv!Fobf2+#Q`&qz8p?xW}hnvt(vojuPI$Z40tTA8=tTV9V-JzjpFr>d^kuAU{AOo} z$>qdOL}y_3+ssdfd|5IXB673)d`dJXq8Q3J7-88W>}T!CKrQTa;T{*o^>n_f%5^W+ z48M~Bes?=`M7NaFC2hRF{g%AQX)Bl2$|);H}FvOqfOU!i`1 z`giT~6J<}wU2nVo-rMBy;Y{l{`DOK4e|to`ew4j(eG_BA7%&F@8w2RsEb*?Ot;T>c zU<@=2@b|$(856}?Fnl_&gb@HZggFTIJWFtnS4 zbaFa;IIZk-LUExw?(ai5xkS)bW55{bG7!n-nEU_f*Ykfj$*zn6W8hCQ;D+fuo#BzR xxAq>6d##5qpe!6$3pOd(gjS4LZp8=CAh3Jh029Sp5Eh922m~5zF$R8=fe+OPN8|ti literal 0 HcmV?d00001 diff --git a/dome9/.idea/.gitignore b/dome9/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/dome9/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/dome9/.idea/aqua_csv_2nucleus.iml b/dome9/.idea/aqua_csv_2nucleus.iml new file mode 100644 index 0000000..eb436f5 --- /dev/null +++ b/dome9/.idea/aqua_csv_2nucleus.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/dome9/.idea/inspectionProfiles/profiles_settings.xml b/dome9/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/dome9/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/dome9/.idea/misc.xml b/dome9/.idea/misc.xml new file mode 100644 index 0000000..0b8b24f --- /dev/null +++ b/dome9/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/dome9/.idea/modules.xml b/dome9/.idea/modules.xml new file mode 100644 index 0000000..e397bde --- /dev/null +++ b/dome9/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/dome9/README.md b/dome9/README.md new file mode 100644 index 0000000..ad198fd --- /dev/null +++ b/dome9/README.md @@ -0,0 +1,25 @@ +This script is a command line tool for easily converting Dome9 csv output into a Nucleus format for easy upload. The script iterates through the Dome9 file, normalizing it into the Nucleus Custom Schema and then writes it to CSV output. The script can also optionally POST the file to your designated Nucleus project. + +Install: + +This project uses Conda to manage the Python virtual environment. + +Conda can be [downloaded here](https://docs.conda.io/projects/conda/en/latest/user-guide/install/download.html) + +The documentation for managing the environment [can be found here](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) + +The process for creating the environment is: +1. `conda env create -f environment.yml` with the environment.yml reference to the environment yaml in the project. +2. `conda activate nucleus-plugin` to activate your virtual environment +3. To deactivate the environment, execute `conda deactivate` + +Usage: +- -i for the input Dome9 file path +- -o for the output file path for the output csv + +**Example:** `python3 dome9_2nucleus_csv.py -i files/dome9_findings.csv -o dome9_2nucleus.csv -# 11000176` + +Optional: +- - -# Project ID of the Nucleus project to which you to POST the output file + +**Note**: If you want to post the file to Nucleus you will need to get an API key from the Nucleus console and update the NUCLEUS_ROOT_URL in the python script. \ No newline at end of file diff --git a/dome9/dome9_2nucleus_csv.py b/dome9/dome9_2nucleus_csv.py new file mode 100644 index 0000000..b6be95d --- /dev/null +++ b/dome9/dome9_2nucleus_csv.py @@ -0,0 +1,140 @@ +# Used for writing to csv +import csv +# Used for arguments +import argparse +# Used to post the file to Nucleus +import requests +from datetime import datetime +import re + +# Enter in the root URL of your Nucleus instance WITHOUT the trailing slash. +# Example https://example.nucleussec.com +NUCLEUS_ROOT_URL = "{Enter root URL of your Nucleus Instance here}" +# Generate an API key through the Nucleus UI +API_KEY = "{Enter your API key from Nucleus here}" + + +def customParser(inputPath, outputPath): + with open(inputPath, 'r', newline='', encoding = 'cp1252') as input_file: + findings = csv.reader(input_file) + + # Create the csv file for writing + with open(outputPath, 'w', newline='\n') as output_file: + + # write headers on output file + csvwriter = csv.writer(output_file, delimiter=',') + csvwriter.writerow(['nucleus_import_version', 'host_name', 'scan_type', 'scan_tool', \ + 'finding_number', 'finding_name', 'finding_severity', 'finding_type', \ + 'finding_description', 'finding_exploitable', 'finding_recommendation', \ + 'finding_result', 'asset_info']) + + # skip first line of csv headers + next(findings) + + # iterate through each row in findings and map to nucleus fields + row = 1 + for finding in findings: + if finding == []: + continue + csv_line = [] + if finding[8] != "": + host_name = finding[8] + else: + host_name = finding[0] + find_num = finding[3] + find_name = finding[3] + " - " + finding[2] + find_sev = finding[4] + find_descr = finding[7] + "\n\n" + finding[5] + find_rec = finding[6] + + # generate asset info + asset_info = "" + asset_info = asset_info + "dome9.account_name:" + finding[0] + asset_info = asset_info + ";dome9.aws_account_id:" + finding[1] + + # create csv line array and write to output file + csv_line = ['1', host_name, 'Host', 'Dome9', find_num, find_name, find_sev, 'Vuln', \ + find_descr, "False", find_rec, "Failed", asset_info] + csvwriter.writerow(csv_line) + row += 1 + + output_file.close() + return output_file + +def get_args(): + parser = argparse.ArgumentParser( + description="For parsing whitesource files to be uploaded into Nucleus. If project ID is specified, will post the Nucleus supported file to Nucleus project.") + + # List arguments. Should only include input file and output file + parser.add_argument('-i', '--inputfile', dest='inputFile', help="Path to whitesource xml file to parse", + required=True) + parser.add_argument('-o', '--outputfile', dest='outputFile', help="Path to csv file output", required=True) + parser.add_argument('-#', '--project_id', dest="project_id", + help="This is the project ID of the Nucleus project to which you want to post. If not specified, this script will only parse the whitesource file for manual upload.") + + # Define the arguments globally for ease of use + global args + + args = parser.parse_args() + + return args + + +# Send the file to Nucleus +def post_to_nucleus(outputfile): + # Enter the ID of the project which you wish to post to here + PROJECT_ID = args.project_id + + # open the file to send + with open(outputfile.name, 'rb') as f: + + # Get the final Nucleus URL to post to + nucleus_url = str(NUCLEUS_ROOT_URL + '/nucleus/api/projects/' + PROJECT_ID + '/scans') + + print("Posted to URL:", nucleus_url) + + # Send file with proper header. Keep note of the project ID you need to send + file_upload = requests.post(nucleus_url, files={outputfile.name: f}, headers={'x-apikey': API_KEY}) + + # Print the response from the server + print(file_upload.content) + + if file_upload.status_code == 404: + + print( + "You probably entered the wrong url. Check to make sure the last slash '/' has been removed from the NUCLEUS_ROOT_URL") + + else: + + pass + + +if __name__ == "__main__": + + # Get the arguments + arguments = get_args() + + # Get the input file to parse + inputPath = arguments.inputFile + + # Get the output file to save to + outputPath = arguments.outputFile + + # Print to cli so user has feedback + print("File received, parsing contents") + + # Start the parsing and csv writing + outputfile = customParser(inputPath, outputPath) + + # If a project ID was specified, send the file to Nucleus + if arguments.project_id: + + print("Parsing complete. Sending to specified Nucleus project") + + # Send the newly created csv file to Nucleus if project id was specified + post_to_nucleus(outputfile) + + # If no project ID was specified, just parse file to Nucleus format for manual file upload + else: + + print("Parsing complete. Success!") diff --git a/dome9/envrionment.yml b/dome9/envrionment.yml new file mode 100644 index 0000000..c3a739f --- /dev/null +++ b/dome9/envrionment.yml @@ -0,0 +1,9 @@ +name: nucleus-plugin + +dependencies: + - python=3.9 + - pip + - pip: + - requests + - numpy + - pandas diff --git a/dome9/files/.DS_Store b/dome9/files/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0