diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..ba8708f5 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,28 @@ +Cobra Changelog +=============== + +Here you can see the full list of changes between each Cobra release. + +Version 2.0.0-alpha.2 +--------------------- + +Released on Sep 06 2017 + +- 修复上传非支持的后缀提示 +- 修复VirtualEnv环境下无法执行 +- 修复grep/find路径位置变动 +- 优化日志等级 +- 优化Docker下路径错误 +- 优化耗时计算 +- 其它细节优化和Bug修复 + +Version 2.0.0-alpha.1 +--------------------- + +Released on Sep 05 2017 + +内测正式版本 + +- 简化安装和使用成本 +- 增加CLI模式 +- 开源扫描规则 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46dc534c..b013ce90 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,9 @@ 仔细描述问题的复现步骤,并提供对应的运行环境信息(Python版本、系统版本) ## 提交代码 -- Fork项目,切换到`develop`分支开发,或新建分支`feature-xxx` +- Fork项目,切换到`develop`分支开发 - 按照PEP8格式 - 所有代码都需要有对应的单元测试用例 - 运行所有测试用例 -- 提交Pull Request \ No newline at end of file +- 提交Pull Request到`develop`分支 +- 等待测试稳定后合并到`master`分支 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3c9b6e7f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:xenial + +COPY . /code/ +WORKDIR /code + +RUN apt-get update && apt-get install -y python-pip curl \ + && apt-get autoremove \ + && apt-get clean \ + && apt-get autoclean \ + && pip install -r requirements.txt \ + && cp config.template config + +EXPOSE 5000 +CMD ["python", "cobra.py", "-H", "0.0.0.0", "-P", "5000"] diff --git a/cobra.py b/cobra.py index cccb2168..aa7f1ca6 100755 --- a/cobra.py +++ b/cobra.py @@ -1,11 +1,13 @@ -#!/usr/bin/python - +#!/usr/bin/env python # -*- coding: utf-8 -*- + import re import sys from cobra import main + if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(main()) + diff --git a/cobra/__init__.py b/cobra/__init__.py index 8c34c662..f624f152 100644 --- a/cobra/__init__.py +++ b/cobra/__init__.py @@ -37,7 +37,7 @@ def main(): try: # arg parse - t1 = time.clock() + t1 = time.time() parser = argparse.ArgumentParser(prog=__title__, description=__introduction__, epilog=__epilog__, formatter_class=argparse.RawDescriptionHelpFormatter) parser_group_scan = parser.add_argument_group('Scan') @@ -80,7 +80,7 @@ def main(): # API call CLI mode a_sid = args.sid cli.start(args.target, args.format, args.output, args.special_rules, a_sid) - t2 = time.clock() + t2 = time.time() logger.info('[INIT] Done! Consume Time:{ct}s'.format(ct=t2 - t1)) except Exception as e: err_msg = unhandled_exception_message() diff --git a/cobra/__version__.py b/cobra/__version__.py index 1b27a50b..a5a44fb7 100644 --- a/cobra/__version__.py +++ b/cobra/__version__.py @@ -7,7 +7,7 @@ __issue_page__ = 'https://github.com/wufeifei/cobra/issues/new' __python_version__ = sys.version.split()[0] __platform__ = platform.platform() -__version__ = '2.0.0-alpha' +__version__ = '2.0.0-alpha.2' __author__ = 'Feei' __author_email__ = 'feei@feei.cn' __license__ = 'MIT License' @@ -22,10 +22,10 @@ Cobra is a static code analysis system that automates the detecting vulnerabilities and security issue.""".format(version=__version__) __epilog__ = """Usage: - {m} -t {td} - {m} -t {td} -r cvi-190001,cvi-190002 - {m} -t {td} -f json -o /tmp/report.json - {m} -t {tg} -f json -o feei@feei.cn - {m} -t {tg} -f json -o http://push.to.com/api - sudo {m} -H 127.0.0.1 -P 80 -""".format(m='./cobra.py', td='tests/vulnerabilities', tg='https://github.com/ethicalhack3r/DVWA') + python {m} -t {td} + python {m} -t {td} -r cvi-190001,cvi-190002 + python {m} -t {td} -f json -o /tmp/report.json + python {m} -t {tg} -f json -o feei@feei.cn + python {m} -t {tg} -f json -o http://push.to.com/api + sudo python {m} -H 127.0.0.1 -P 80 +""".format(m='cobra.py', td='tests/vulnerabilities', tg='https://github.com/ethicalhack3r/DVWA') diff --git a/cobra/api.py b/cobra/api.py index ddb24d3a..d021cb29 100644 --- a/cobra/api.py +++ b/cobra/api.py @@ -42,6 +42,8 @@ q = queue.Queue() app = Flask(__name__, static_folder='templates/asset') +running_host = '0.0.0.0' +running_port = 5000 def producer(task): @@ -204,7 +206,7 @@ def post(): code, result = 1001, {'sid': a_sid} return {'code': code, 'result': result} else: - return {'code': 1002, 'msg': "This extension can't support!"} + return {'code': 1002, 'result': "This extension can't support!"} class ResultData(Resource): @@ -309,7 +311,8 @@ def summary(): return render_template(template_name_or_list='index.html', key=key) - status_url = request.url_root + 'api/status' + status_url = 'http://{host}:{port}/api/status'.format(host=running_host, port=running_port) + logger.critical(status_url) post_data = { 'key': key, 'sid': a_sid, @@ -481,6 +484,9 @@ def start(host, port, debug): i.start() try: + global running_port, running_host + running_host = host if host != '0.0.0.0' else '127.0.0.1' + running_port = port app.run(debug=debug, host=host, port=int(port), threaded=True, processes=1) except socket.error as v: if v.errno == errno.EACCES: diff --git a/cobra/cve_parse.py b/cobra/cve.py similarity index 91% rename from cobra/cve_parse.py rename to cobra/cve.py index e43c5419..6aa2bfbd 100644 --- a/cobra/cve_parse.py +++ b/cobra/cve.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- + """ - cobra - ~~~~~ + CVE + ~~~ - Implements cobra main + Implements CVE Rules Parser :author: BlBana <635373043@qq.com> :homepage: https://github.com/wufeifei/cobra @@ -241,9 +242,9 @@ def log_result(self): for cve_child in self._scan_result[module_]: cve_id = cve_child level = self._scan_result[module_][cve_id] - logger.warning('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) + logger.debug('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) count = len(self._scan_result[module_]) - logger.warning('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') + logger.debug('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') def get_scan_result(self): return self._scan_result @@ -337,7 +338,7 @@ def store(results): for module_ in results[0]: for cve_id, cve_level in results[0][module_].items(): cve_path = results[1] - cve_vul = parse_math(cve_path, cve_id, cve_level, module_) + cve_vul = parse_math(cve_path, cve_id, cve_level, module_, target_directory) cve_vuls.append(cve_vul) else: logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!') @@ -371,11 +372,12 @@ def scan_single(target_directory, cve_path): return cve.get_scan_result(), cve_path -def parse_math(cve_path, cve_id, cve_level, module_): +def parse_math(cve_path, cve_id, cve_level, module_, target_directory): + flag = 0 + file_path = 'unkown' mr = VulnerabilityResult() module_name, module_version = module_.split(':') cvi = cve_path.lower().split('cvi-')[1][:6] - rule_name = '引用了存在漏洞的三方组件' if cve_level == 'LOW': cve_level = 2 @@ -385,13 +387,37 @@ def parse_math(cve_path, cve_id, cve_level, module_): elif cve_level == 'HIGH': cve_level = 8 - mr.language = cve_id + for root, dirs, filenames in os.walk(target_directory): + for filename in filenames: + if filename == 'pom.xml' and flag != 2: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 1 + + elif filename == 'requirements.txt' and flag != 1: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 2 + + if flag != 0: + mr.file_path = file_path + + else: + mr.file_path = 'unkown' + mr.language = '*' mr.id = cvi - mr.rule_name = rule_name + mr.rule_name = cve_id mr.level = cve_level - mr.file_path = module_name mr.line_number = 1 + mr.analysis = 'Dependencies Matched(依赖匹配)' mr.code_content = module_name + ':' + module_version + mr.solution = """ + 三方依赖**""" + module_name + """:""" + module_version + """**存在CVE漏洞,CVE漏洞编号为: **""" + cve_id + """** + ## 安全风险 + + ## 安全修复 + 请根据对应厂商公告,及时更新三方依赖至安全版本 + """ logger.debug('[CVE {i}] {r}:Find {n}:{v} have vul {c} and level is {l}'.format(i=mr.id, r=mr.rule_name, n=mr.file_path, v=mr.line_number, diff --git a/cobra/dependencies.py b/cobra/dependencies.py index 53d65284..ed7ddb98 100644 --- a/cobra/dependencies.py +++ b/cobra/dependencies.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ - cobra - ~~~~~ + dependencies + ~~~~~~~~~~~~ - Implements cobra main + Implements Dependencies Check :author: BlBana <635373043@qq.com> :homepage: https://github.com/wufeifei/cobra diff --git a/cobra/engine.py b/cobra/engine.py index d09604e8..4aae64e7 100644 --- a/cobra/engine.py +++ b/cobra/engine.py @@ -26,7 +26,7 @@ from .result import VulnerabilityResult from .cast import CAST from .parser import scan_parser -from .cve_parse import scan_cve +from .cve import scan_cve from prettytable import PrettyTable @@ -70,7 +70,7 @@ def list(self, data=None): result = f.readline() return json.loads(result) else: - with open(file_path, 'r+') as f: # w+ causes a file reading bug + with open(file_path, 'r+') as f: # w+ causes a file reading bug fcntl.flock(f, fcntl.LOCK_EX) result = f.read() if result == '': @@ -132,7 +132,13 @@ def score2level(score): if level is None: return 'Unknown' else: - return '{l}-{s}: {ast}'.format(l=level[:1], s=score, ast='☆' * score) + if score < 10: + score_full = '0{s}'.format(s=score) + else: + score_full = score + + a = '{s}{e}'.format(s=score * '■', e=(10 - score) * '□') + return '{l}-{s}: {ast}'.format(l=level[:1], s=score_full, ast=a) def scan_single(target_directory, single_rule): @@ -194,12 +200,12 @@ def store(result): # print data = [] - table = PrettyTable(['#', 'CVI', 'VUL', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level-Score', 'Target-File:Line-Number/Module:Version', 'Commit(Author/Time)', 'Source Code Content']) + table = PrettyTable(['#', 'CVI', 'VUL', 'Rule', 'Lang', 'Level-Score', 'Target', 'Commit(Time, Author)', 'Source Code Content', 'Analysis']) table.align = 'l' trigger_rules = [] for idx, x in enumerate(find_vulnerabilities): trigger = '{fp}:{ln}'.format(fp=x.file_path, ln=x.line_number) - commit = u'@{author},{time}'.format(author=x.commit_author, time=x.commit_time) + commit = u'{time}, @{author}'.format(author=x.commit_author, time=x.commit_time) level = score2level(x.level) cvi = x.id[0:3] if cvi in vulnerabilities: @@ -207,10 +213,10 @@ def store(result): else: cvn = 'Unknown' try: - code_content = x.code_content[:100].strip() + code_content = x.code_content[:50].strip() except AttributeError as e: code_content = x.code_content.decode('utf-8')[:100].strip() - row = [idx + 1, x.id, cvn, x.rule_name, x.language, level, trigger, commit, code_content] + row = [idx + 1, x.id, cvn, x.rule_name, x.language, level, trigger, commit, code_content, x.analysis] data.append(row) table.add_row(row) if x.id not in trigger_rules: @@ -334,12 +340,13 @@ def process(self): continue is_test = False try: - is_vulnerability, status_code = Core(self.target_directory, vulnerability, self.sr, 'project name', ['whitelist1', 'whitelist2'], test=is_test, index=index).scan() + is_vulnerability, reason = Core(self.target_directory, vulnerability, self.sr, 'project name', ['whitelist1', 'whitelist2'], test=is_test, index=index).scan() if is_vulnerability: - logger.debug('[CVI-{cvi}] [RET] Found {code}'.format(cvi=self.sr['id'], code=status_code)) + logger.debug('[CVI-{cvi}] [RET] Found {code}'.format(cvi=self.sr['id'], code=reason)) + vulnerability.analysis = reason self.rule_vulnerabilities.append(vulnerability) else: - logger.debug('Not vulnerability: {code}'.format(code=status_code)) + logger.debug('Not vulnerability: {code}'.format(code=reason)) except Exception: raise logger.debug('[CVI-{cvi}] {vn} Vulnerabilities: {count}'.format(cvi=self.sr['id'], vn=self.sr['name'], count=len(self.rule_vulnerabilities))) @@ -544,43 +551,47 @@ def scan(self): :return: is_vulnerability, code """ self.method = 0 + self.code_content = self.code_content + if len(self.code_content) > 512: + self.code_content = self.code_content[:500] + self.status = self.status_init + self.repair_code = self.repair_code_init if self.is_white_list(): logger.debug("[RET] Whitelist") - return False, 5001 + return False, 'Whitelists(白名单)' if self.is_special_file(): logger.debug("[RET] Special File") - return False, 5002 + return False, 'Special File(特殊文件)' if self.is_test_file(): logger.debug("[CORE] Test File") if self.is_annotation(): logger.debug("[RET] Annotation") - return False, 5004 + return False, 'Annotation(注释)' if self.rule_match_mode == const.mm_find_extension: # # Find-Extension # Match(extension) -> Done # - found_vul = True + return True, 'FIND-EXTENSION(后缀查找)' elif self.rule_match_mode == const.mm_regex_only_match: # # Regex-Only-Match # Match(regex) -> Repair -> Done # logger.debug("[CVI-{cvi}] [ONLY-MATCH]".format(cvi=self.cvi)) - found_vul = True if self.rule_match2 is not None: ast = CAST(self.rule_match, self.target_directory, self.file_path, self.line_number, self.code_content) is_match, data = ast.match(self.rule_match2, self.rule_match2_block) if is_match: logger.debug('[CVI-{cvi}] [MATCH2] True'.format(cvi=self.cvi)) - return True, 1001 + return True, 'REGEX-ONLY-MATCH+MATCH2(正则仅匹配+二次匹配)' else: logger.debug('[CVI-{cvi}] [MATCH2] False'.format(cvi=self.cvi)) - return False, 1002 + return False, 'REGEX-ONLY-MATCH+Not matched2(未匹配到二次规则)' if self.rule_repair is not None: logger.debug('[VERIFY-REPAIR]') @@ -589,10 +600,12 @@ def scan(self): if is_repair: # fixed logger.debug('[CVI-{cvi}] [RET] Vulnerability Fixed'.format(cvi=self.cvi)) - return False, 1002 + return False, 'REGEX-ONLY-MATCH+Vulnerability-Fixed(漏洞已修复)' else: logger.debug('[CVI-{cvi}] [REPAIR] [RET] Not fixed'.format(cvi=self.cvi)) - found_vul = True + return True, 'REGEX-ONLY-MATCH+NOT FIX(未修复)' + else: + return True, 'REGEX-ONLY-MATCH(正则仅匹配+无修复规则)' else: # # Function-Param-Controllable @@ -604,12 +617,9 @@ def scan(self): # Match(regex) -> Match2(regex) -> Param-Controllable -> Repair -> Done # logger.debug('[CVI-{cvi}] match-mode {mm}'.format(cvi=self.cvi, mm=self.rule_match_mode)) - found_vul = False - result = [] if self.file_path[-3:].lower() == 'php': try: ast = CAST(self.rule_match, self.target_directory, self.file_path, self.line_number, self.code_content) - # Match2 if self.rule_match_mode == const.mm_function_param_controllable: rule_match = self.rule_match.strip('()').split('|') logger.debug('[RULE_MATCH] {r}'.format(r=rule_match)) @@ -620,16 +630,16 @@ def scan(self): logger.debug('[AST] [RET] {c}'.format(c=result)) if len(result) > 0: if result[0]['code'] == 1: # 函数参数可控 - return True, 1001 + return True, 'FUNCTION-PARAM-CONTROLLABLE(函数入参可控)' if result[0]['code'] == 2: # 函数为敏感函数 - return True, 1001 + return False, 'FUNCTION-PARAM-CONTROLLABLE(函数入参来自所在函数)' if result[0]['code'] == 0: # 漏洞修复 - return False, 1002 + return False, 'FUNCTION-PARAM-CONTROLLABLE+Vulnerability-Fixed(漏洞已修复)' if result[0]['code'] == -1: # 函数参数不可控 - return False, 1002 + return False, 'FUNCTION-PARAM-CONTROLLABLE(入参不可控)' logger.debug('[AST] [CODE] {code}'.format(code=result[0]['code'])) else: @@ -638,14 +648,16 @@ def scan(self): logger.warning(traceback.format_exc()) raise + # Match2 if self.rule_match2 is not None: is_match, data = ast.match(self.rule_match2, self.rule_match2_block) if is_match: logger.debug('[CVI-{cvi}] [MATCH2] True'.format(cvi=self.cvi)) - return True, 1001 + return True, 'FPC+MATCH2(函数入参可控+二次匹配)' else: logger.debug('[CVI-{cvi}] [MATCH2] False'.format(cvi=self.cvi)) - return False, 1002 + return False, 'FPC+NOT-MATCH2(函数入参可控+二次未匹配)' + # Param-Controllable param_is_controllable, data = ast.is_controllable_param() if param_is_controllable: @@ -655,24 +667,13 @@ def scan(self): if is_repair: # fixed logger.debug('[CVI-{cvi}] [REPAIR] Vulnerability Fixed'.format(cvi=self.cvi)) - return False, 1002 + return False, 'Vulnerability-Fixed(漏洞已修复)' else: logger.debug('[CVI-{cvi}] [REPAIR] [RET] Not fixed'.format(cvi=self.cvi)) - found_vul = True + return True, 'MATCH+REPAIR(匹配+未修复)' else: logger.debug('[CVI-{cvi}] [PARAM-CONTROLLABLE] Param Not Controllable'.format(cvi=self.cvi)) - return False, 4002 + return False, 'Param-Not-Controllable(参数不可控)' except Exception as e: logger.debug(traceback.format_exc()) - return False, 4004 - - if found_vul: - self.code_content = self.code_content - if len(self.code_content) > 512: - self.code_content = self.code_content[:500] - self.status = self.status_init - self.repair_code = self.repair_code_init - return True, 1001 - else: - logger.debug("[CVI-{cvi}] [DONE] Not found vulnerability".format(cvi=self.cvi)) - return False, 4002 + return False, 'Exception' diff --git a/cobra/export.py b/cobra/export.py index 2e7adc91..4648dd4e 100644 --- a/cobra/export.py +++ b/cobra/export.py @@ -2,7 +2,7 @@ """ export - ~~~~~~~~~~~~~~~~~~~ + ~~~~~~ Export scan result to files or console diff --git a/cobra/git_projects.py b/cobra/git_projects.py index b3c8b20c..8d0e0ff8 100644 --- a/cobra/git_projects.py +++ b/cobra/git_projects.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- + """ cobra ~~~~~ @@ -17,6 +17,7 @@ import threading from .log import logger from .config import code_path, Config + try: # Python 3 import queue @@ -36,7 +37,7 @@ def start(): result_path = code_path + '/result_sid' fi = open(result_path, 'w+') for i in range(int(pages)): - q_pages.put(i+1) + q_pages.put(i + 1) for i in range(10): thread = threading.Thread(target=get_git_urls, args=(url, private_token, cobra_ip, key, q_pages, fi)) @@ -74,7 +75,7 @@ def get_git_urls(url, private_token, cobra_ip, key, q_pages, fi): git_branch = data[j]['default_branch'] if git_branch is not None: - request_url = git_url+':'+git_branch + request_url = git_url + ':' + git_branch else: request_url = git_url diff --git a/cobra/parser.py b/cobra/parser.py index ccbeef9e..f3dc3db3 100644 --- a/cobra/parser.py +++ b/cobra/parser.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ - cobra - ~~~~~ + parser + ~~~~~~ - Implements cobra main + Implements Code Parser :author: BlBana <635373043@qq.com> :homepage: https://github.com/wufeifei/cobra diff --git a/cobra/result.py b/cobra/result.py index a442830e..a10b1e9d 100644 --- a/cobra/result.py +++ b/cobra/result.py @@ -17,6 +17,7 @@ class VulnerabilityResult: def __init__(self): self.id = '' self.file_path = None + self.analysis = '' self.rule_name = '' self.language = '' diff --git a/cobra/scheduler/__init__.py b/cobra/scheduler/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cobra/scheduler/report.js b/cobra/scheduler/report.js deleted file mode 100644 index 1e6ee614..00000000 --- a/cobra/scheduler/report.js +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Use PhantomJS to capture the report page - * - * @author Feei - * @homepage https://github.com/wufeifei/cobra - * @license MIT, see LICENSE for more details. - * @copyright Copyright (c) 2017 Feei. All rights reserved - */ -"use strict"; -/** - * http://phantomjs.org/api/webpage/ - * - * Web page api - */ -var page = require('webpage').create(); -/** - * http://phantomjs.org/api/fs/ - * - * file system api - */ -var fs = require('fs'); -/** - * http://phantomjs.org/api/system/ - * - * system - */ -var system = require('system'); - -/** - * filename generator helper - * @param viewport - * @param tt time type - * @returns {string} - */ -function getFileName(viewport, tt) { - var d = new Date(); - var date = [ - d.getUTCFullYear(), - d.getUTCMonth() + 1, - d.getUTCDate() - ]; - var time = [ - d.getHours() <= 9 ? '0' + d.getHours() : d.getHours(), - d.getMinutes() <= 9 ? '0' + d.getMinutes() : d.getMinutes(), - d.getSeconds() <= 9 ? '0' + d.getSeconds() : d.getSeconds(), - d.getMilliseconds() - ]; - var resolution = viewport.width + "x" + viewport.height; - return tt + '_' + date.join('-') + '_' + time.join('-') + "_" + resolution + '.png'; -} - - -/** - * Read the Cobra config - * @type {string} - */ -if (system.args.length < 3 || system.args[0] in ['w', 'm', 'q']) { - console.log('Usage: report.js = 4) { - var month = system.args[3]; - domain += '&month=' + month; -} -var file = 'reports/' + getFileName(page.viewportSize, tt); -/** - * Capture - */ -page.open(domain, function (status) { - if (status !== 'success') { - console.log('Critical: Unable to load the address!'); - phantom.exit(1); - } else { - /** - * Draw chart need time - */ - window.setTimeout(function () { - page.render(file); - console.log('Success: ' + file); - phantom.exit(); - }, 1000); - } -}); \ No newline at end of file diff --git a/cobra/scheduler/report.py b/cobra/scheduler/report.py deleted file mode 100644 index e77cc3c0..00000000 --- a/cobra/scheduler/report.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - scheduler.report - ~~~~~~~~~~~~~~~~ - - Implements automation report Cobra data - - :author: Feei - :homepage: https://github.com/wufeifei/cobra - :license: MIT, see LICENSE for more details. - :copyright: Copyright (c) 2017 Feei. All rights reserved -""" -import os -import subprocess -import base64 -import datetime -from cobra.utils.log import logging -from cobra.utils.config import Config - -import smtplib -from smtplib import SMTPException -from email.mime.text import MIMEText -from email.mime.multipart import MIMEMultipart - -logging = logging.getLogger(__name__) - -phantomjs = '/usr/local/bin/phantomjs' -time_types = ['w', 'm', 'q'] -time_type_des = { - 'w': '周', - 'm': '月', - 'q': '季' -} - - -class Report(object): - def __init__(self, time_type, month=None): - if time_type not in time_types: - logging.critical('Time type exception') - return - - self.time_type_de = time_type_des[time_type] - - # mail - mark = '' - if month is None: - c_month = int(datetime.datetime.today().strftime("%m")) - else: - c_month = int(month) - - if time_type == 'w': - c_week = int(datetime.datetime.today().strftime("%U")) - mark = 'W{week}'.format(week=c_week) - elif time_type == 'm': - mark = 'M{month}'.format(month=c_month) - elif time_type == 'q': - c_quarter = 0 - if c_month in [1, 2, 3]: - c_quarter = 1 - elif c_month in [4, 5, 6]: - c_quarter = 2 - elif c_month in [7, 8, 9]: - c_quarter = 3 - elif c_month in [10, 11, 12]: - c_quarter = 4 - mark = 'Q{quarter}'.format(quarter=c_quarter) - self.subject = '[Cobra] 代码安全{0}报({mark})'.format(self.time_type_de, mark=mark) - self.user = Config('email', 'user').value - self.name = Config('email', 'name').value - self.to = Config('report', 'to').value - self.host = Config('email', 'host').value - self.port = Config('email', 'port').value - self.password = Config('email', 'password').value - - self.param = [phantomjs, os.path.join(Config().project_directory, 'scheduler', 'report.js'), Config().project_directory, time_type] - if month is not None: - self.param.append(month) - - def run(self): - capture = self.capture() - if capture is False: - logging.critical('Capture failed') - return False - - # send notification - if self.notification(capture): - return True - else: - logging.critical('Notification failed') - return False - - def capture(self): - """ - Use PhantomJS to capture report page - :return: boolean - """ - capture = None - p = subprocess.Popen(self.param, stdout=subprocess.PIPE) - result, err = p.communicate() - if 'Critical' in result: - logging.critical('Capture exception') - return False - lines = result.split('\n') - for l in lines: - if 'reports' in l: - capture = l.split(':')[1].strip() - - if capture is None: - logging.critical('get capture image file failed') - return False - else: - return os.path.join(Config().project_directory, capture) - - def notification(self, capture_path): - """ - Email notification - :param capture_path: - :return: boolean - """ - msg = MIMEMultipart() - msg['Subject'] = self.subject - msg['From'] = '{0}<{1}>'.format(self.name, self.user) - msg['To'] = self.to - - with open(capture_path, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()) - - text = MIMEText(''.format(encoded_string), 'html') - msg.attach(text) - - try: - s = smtplib.SMTP(self.host, self.port) - s.ehlo() - s.starttls() - s.ehlo() - s.login(self.user, self.password) - s.sendmail(self.user, self.to, msg.as_string()) - s.quit() - return True - except SMTPException: - logging.critical('Send mail failed') - return False diff --git a/cobra/scheduler/scan.py b/cobra/scheduler/scan.py deleted file mode 100644 index 9870aba0..00000000 --- a/cobra/scheduler/scan.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - scheduler.scan - ~~~~~~~~~~~~~~ - - Implements periodic scan job - - :author: Feei - :homepage: https://github.com/wufeifei/cobra - :license: MIT, see LICENSE for more details. - :copyright: Copyright (c) 2017 Feei. All rights reserved -""" -import json -import requests -from cobra.utils.log import logging -from cobra.utils import common, config -from cobra.app.models import CobraProjects - -logging = logging.getLogger(__name__) - - -class Scan(object): - def __init__(self): - domain = '{0}:{1}'.format(config.Config('cobra', 'host').value, config.Config('cobra', 'port').value) - self.api = 'http://' + domain + '/api/{0}' - self.headers = {"Content-Type": "application/json"} - self.key = common.md5('CobraAuthKey') - self.branch = 'master' - - def all(self): - projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all() - for project in projects: - payload = json.dumps({ - "key": self.key, - "target": project.repository, - "branch": self.branch - }) - - try: - response = requests.post(self.api.format('add'), data=payload, headers=self.headers) - response_json = response.json() - logging.info(project.repository, response_json) - except (requests.ConnectionError, requests.HTTPError) as e: - logging.critical("API Add failed: {0}".format(e)) diff --git a/cobra/templates/index.html b/cobra/templates/index.html index 3eb311fd..721d7993 100644 --- a/cobra/templates/index.html +++ b/cobra/templates/index.html @@ -261,6 +261,7 @@

Drag and drop upload

get_status(); } else { alert(result.result.result); + location.reload(); } }, start: function (e) { @@ -283,4 +284,4 @@

Drag and drop upload

- \ No newline at end of file + diff --git a/cobra/templite.py b/cobra/templite.py index de14580b..ddfc92c6 100644 --- a/cobra/templite.py +++ b/cobra/templite.py @@ -2,7 +2,7 @@ """ templite - ~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~ A simple template engine diff --git a/cobra/utils.py b/cobra/utils.py index d932e9e2..44887a3f 100644 --- a/cobra/utils.py +++ b/cobra/utils.py @@ -395,10 +395,28 @@ def get_safe_ex_string(ex, encoding=None): class Tool: def __init__(self): + # `grep` (`ggrep` on Mac) - self.grep = '/bin/grep' + if os.path.isfile('/bin/grep'): + self.grep = '/bin/grep' + elif os.path.isfile('/usr/bin/grep'): + self.grep = '/usr/bin/grep' + elif os.path.isfile('/usr/local/bin/grep'): + self.grep='/usr/local/bin/grep' + else: + self.grep = 'grep' + + # `find` (`gfind` on Mac) - self.find = '/bin/find' + if os.path.isfile('/bin/find'): + self.find = '/bin/find' + elif os.path.isfile('/usr/bin/find'): + self.find = '/usr/bin/find' + elif os.path.isfile('/usr/local/bin/find'): + self.find='/usr/local/bin/find' + else: + self.find = 'find' + if 'darwin' == sys.platform: ggrep = '' @@ -427,10 +445,10 @@ def secure_filename(filename): _filename_utf8_strip_re = re.compile(u"[^\u4e00-\u9fa5A-Za-z0-9_.\-\+]") _windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', 'COM4', 'LPT1', 'LPT2', 'LPT3', 'PRN', 'NUL') - if PY2: - text_type = unicode - else: - text_type = str + try: + text_type = unicode # Python 2 + except NameError: + text_type = str # Python 3 if isinstance(filename, text_type): from unicodedata import normalize diff --git a/docs/api.md b/docs/api.md index 273f9f50..5a059a67 100644 --- a/docs/api.md +++ b/docs/api.md @@ -55,7 +55,7 @@ # 完整的例子 ## 启动HTTP服务 ```bash -sudo ./cobra.py -H 127.0.0.1 -P 80 +sudo python cobra.py -H 127.0.0.1 -P 80 ``` ## 添加扫描任务 diff --git a/docs/cli.md b/docs/cli.md index 6f5aec2a..60871a92 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,36 +3,36 @@ ## Examples(使用例子) ```bash # 扫描一个文件夹的代码 -$ ./cobra.py -t tests/vulnerabilities +$ python cobra.py -t tests/vulnerabilities # 扫描一个Git项目代码 -$ ./cobra.py -t https://github.com/wufeifei/grw.git +$ python cobra.py -t https://github.com/wufeifei/grw.git # 扫描一个文件夹,并将扫描结果导出为JSON文件 -$ ./cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json +$ python cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json # 扫描一个Git项目,并将扫描结果JSON文件推送到API上 -$ ./cobra.py -f json -o http://push.to.com/api -t https://github.com/wufeifei/vc.git +$ python cobra.py -f json -o http://push.to.com/api -t https://github.com/wufeifei/vc.git # 扫描一个Git项目,并将扫描结果JSON文件发送到邮箱中 -$ ./cobra.py -f json -o feei@feei.cn -t https://github.com/wufeifei/vc.git +$ python cobra.py -f json -o feei@feei.cn -t https://github.com/wufeifei/vc.git # 扫描一个文件夹代码的某两种漏洞 -$ ./cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 +$ python cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 # 开启一个Cobra HTTP Server,然后可以使用API接口来添加扫描任务 -$ ./cobra.py -H 127.0.0.1 -P 80 +$ python cobra.py -H 127.0.0.1 -P 80 # 查看版本 -$ ./cobra.py --version +$ python cobra.py --version # 查看帮助 -$ ./cobra.py --help +$ python cobra.py --help ``` ## Help(帮助) ```bash -➜ cobra git:(master) ✗ ./cobra.py --help +➜ cobra git:(master) ✗ python cobra.py --help usage: cobra [-h] [-t ] [-f ] [-o ] [-r ] [-d] [-sid SID] [-H ] [-P ] @@ -68,12 +68,12 @@ RESTful: REST-JSON API Service Port Usage: - ./cobra.py -t tests/vulnerabilities - ./cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 - ./cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json - ./cobra.py -t https://github.com/wufeifei/vc.git -f json -o feei@feei.cn - ./cobra.py -t https://github.com/wufeifei/vc.git -f json -o http://push.to.com/api - sudo ./cobra.py -H 127.0.0.1 -P 80 + python cobra.py -t tests/vulnerabilities + python cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 + python cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json + python cobra.py -t https://github.com/ethicalhack3r/DVWA -f json -o feei@feei.cn + python cobra.py -t https://github.com/ethicalhack3r/DVWA -f json -o http://push.to.com/api + sudo python cobra.py -H 127.0.0.1 -P 80 ``` --- 下一章:[API模式使用方法](https://wufeifei.github.io/cobra/api) \ No newline at end of file diff --git a/docs/contributors.md b/docs/contributors.md index 6d10ecb7..b70d84dc 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -9,6 +9,7 @@ | [LiGhT1EsS](https://github.com/LiGhT1EsS) | [https://lightless.me](https://lightless.me) | root@lightless.me | | [40huo](https://github.com/40huo) | [https://www.40huo.cn](https://www.40huo.cn) | i@40huo.cn | | [BlBana](https://github.com/BlBana) | [http://drops.blbana.cc](http://drops.blbana.cc) | 635373043@qq.com | +| [Bre4k](https://github.com/bk7477890) | http://bre4k.cn | bk7477890@outlook.com | ## 主要贡献者 braveghz、M2shad0w、alioth310、l4yn3、boltomli、JoyChou93 diff --git a/docs/index.md b/docs/index.md index dd793f80..2b23ce2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -91,4 +91,5 @@ - [程序目录结构](https://wufeifei.github.io/cobra/tree) - 贡献代码 - [单元测试](https://wufeifei.github.io/cobra/test) - - [贡献者](https://wufeifei.github.io/cobra/contributors) \ No newline at end of file + - [贡献者](https://wufeifei.github.io/cobra/contributors) + - [如何贡献代码?](https://github.com/wufeifei/cobra/blob/master/CONTRIBUTING.md) \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 7a164a96..54bee770 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -29,7 +29,7 @@ brew install grep findutils ```bash git clone https://github.com/wufeifei/cobra.git && cd cobra pip install -r requirements.txt -./cobra.py --help +python cobra.py --help ``` --- diff --git a/docs/rule_flow.md b/docs/rule_flow.md index 45ad3a7c..c32066c8 100644 --- a/docs/rule_flow.md +++ b/docs/rule_flow.md @@ -7,7 +7,7 @@ #### 2. 编写漏洞代码`tests/vulnerabilities/v.language` 编写实际可能出现的业务场景代码(只需编写一处即可)。 -#### 3. 测试规则扫描`./cobra.py -t tests/vulnerabilities/` +#### 3. 测试规则扫描`python cobra.py -t tests/vulnerabilities/` 测试扫描结果 --- diff --git a/rules/CVI-160002.xml b/rules/CVI-160002.xml index 6faab099..2afb0fb7 100644 --- a/rules/CVI-160002.xml +++ b/rules/CVI-160002.xml @@ -51,6 +51,6 @@ 尽量不要使用拼接的SQL语句。若不得不使用,尽量不要用户可控SQL语句的参数。 若必须用户可控,请对用户输入的参数进行严格的限制和过滤。 - + diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 340ecaf0..9d5cd0c7 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -12,15 +12,17 @@ :copyright: Copyright (c) 2017 Feei. All rights reserved """ -import requests import json import multiprocessing -import time import os import shutil import socket -from cobra.config import project_directory +import time + +import requests + from cobra.api import start +from cobra.config import project_directory, running_path p = multiprocessing.Process(target=start, args=('127.0.0.1', 5000, False)) p.start() @@ -30,17 +32,39 @@ template_path = os.path.join(project_directory, 'config.template') shutil.copyfile(template_path, config_path) +a_sid = '' +s_sid = '' + def test_add_job(): url = "http://127.0.0.1:5000/api/add" post_data = { "key": "your_secret_key", - "target": ["tests/vulnerabilities"], + "target": [os.path.join(project_directory, 'tests/vulnerabilities')] } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) + result = json.loads(re.text) + + global a_sid + a_sid = result.get('result').get('sid') + + a_sid_file = os.path.join(running_path, '{sid}_list'.format(sid=a_sid)) + + # wait writing scan_list + while True: + with open(a_sid_file, 'r') as f: + scan_list = json.load(f) + print(scan_list) + if len(scan_list.get('sids')) > 0: + break + time.sleep(0.1) + + global s_sid + s_sid = list(scan_list.get('sids').keys())[0] + assert "1001" in re.text assert "Add scan job successfully" in re.text assert "sid" in re.text @@ -50,15 +74,15 @@ def test_job_status(): url = "http://127.0.0.1:5000/api/status" post_data = { "key": "your_secret_key", - "sid": 24, + "sid": a_sid, } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert "1004" in re.text + assert "1001" in re.text assert "msg" in re.text - assert "sid" in re.text + assert a_sid in re.text assert "status" in re.text assert "report" in re.text @@ -66,28 +90,41 @@ def test_job_status(): def test_result_data(): url = 'http://127.0.0.1:5000/api/list' post_data = { - 'sid': 24, + 'sid': s_sid, } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert '1002' in re.text - assert 'No such target' in re.text + + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + if os.path.exists(s_sid_file): + assert '1001' in re.text + assert 'result' in re.text + assert 'rule_filter' in re.text + else: + assert '1002' in re.text + assert 'No such target' in re.text def test_result_detail(): url = 'http://127.0.0.1:5000/api/detail' post_data = { - 'sid': 'abcdeft', - 'file_path': 'setup.py', + 'sid': s_sid, + 'file_path': 'v.php', } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert '1002' in re.text - assert 'No such target' in re.text + + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + if os.path.exists(s_sid_file): + assert '1001' in re.text + assert 'file_content' in re.text + else: + assert '1002' in re.text + assert 'No such target' in re.text def test_index(): @@ -105,23 +142,15 @@ def test_close_api(): p.terminate() p.join() - # wait for scan process - while True: - cobra_process = os.popen('ps aux | grep python').read() - cobra_process_num = len(cobra_process.strip().split('\n')) - if cobra_process_num <= 3: - # grep python - # sh -c ps aux | grep python - # python pytest - break + # wait for scan data + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + while not os.path.exists(s_sid_file): time.sleep(1) - # whether port 5000 is closed + # wait for port closed s = socket.socket() s.settimeout(0.5) - try: - assert s.connect_ex(('localhost', 5000)) != 0 - finally: - s.close() + while s.connect_ex(('localhost', 5000)) == 0: + time.sleep(0.5) assert not os.path.exists(config_path) diff --git a/tests/test_cve_parse.py b/tests/test_cve_parse.py index 57e79e7d..2be6d05e 100644 --- a/tests/test_cve_parse.py +++ b/tests/test_cve_parse.py @@ -14,8 +14,8 @@ """ import pytest import xml.etree.ElementTree as eT -from cobra.cve_parse import * -from cobra.cve_parse import CveParse, project_directory +from cobra.cve import * +from cobra.cve import CveParse, project_directory try: from configparser import ConfigParser, NoSectionError except ImportError: