Skip to content

Commit

Permalink
Merge pull request #78 from mpast/feat-improve-performance-scan-multi…
Browse files Browse the repository at this point in the history 
…line

Feat improve performance scan multiline
  • Loading branch information
@mpast
mpast committed Dec 26, 2022
2 parents 963f914 + 1e5a86e commit 21d4029
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 87 deletions.
182 changes: 108 additions & 74 deletions app/analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def analyze_apk(task, scan_id):
task.update_state(state = 'STARTED',
meta = {'current': scan.progress, 'total': 100, 'status': scan.status})
logger.debug(scan.status)
certificates = get_info_certificate(a, scan)
get_info_certificate(a, scan)
if (settings.VIRUSTOTAL_ENABLED):
scan.status = 'Getting info of VT'
scan.progress = 15
Expand Down Expand Up @@ -98,7 +98,7 @@ def analyze_apk(task, scan_id):
meta = {'current': scan.progress, 'total': 100, 'status': scan.status})
scan.save()
logger.debug(scan.status)
findings = get_tree_dir(scan)
get_tree_dir(scan)
scan.status = 'Finished'
scan.progress = 100
scan.finished_on = datetime.now()
Expand Down Expand Up @@ -231,94 +231,128 @@ def get_tree_dir(scan):
if (extension == '.java' or extension == '.kt' or extension == '.xml'):
try:
prev_line = ''
for i, line in enumerate(open(fname, mode="r", encoding="utf-8")):
find_patterns(i + 1, prev_line, line, fname, dir, scan)
prev_line = line
i = 0
f = open(fname, mode="r", encoding="utf-8")
content = f.read()
f.close()
find_patterns(i + 1, prev_line, content, fname, dir, scan)
except Exception as e:
logger.error('ERROR {} {}'.format(e, fname))
if (filename == 'AndroidManifest.xml'):
get_info_file(scan, fname, dir)
else:
get_info_file(scan, fname, dir)

def get_position(match):
span = match.span()
if span[0] == 0:
span = list(span)
span[0] = 1
return tuple(span)

def get_match_lines(content, position):
lines = ''
c = 0
end = False
beginline = 0
for i, line in enumerate(content.split('\n'), 1):
c += len(line) + 1
if c >= position[0] and c >= position[1] and not end: # only one line
return (i, line)
elif c >= position[0] and not end:
# multiple lines
beginline = i
if (c >= position[1]):
end = True
lines += line + '\n'
if c >= position[1] and end:
# endline
return (beginline, lines)

def find_patterns(i, prev_line, line, name, dir, scan):
patterns = Pattern.objects.filter(active=True)
url = ''
m = ''
for p in patterns:
pattern = re.compile(p.pattern, re.IGNORECASE)
pattern = re.compile(p.pattern, re.MULTILINE)
try:
for match in re.finditer(pattern, line):
type = ''
match_str = match.group()
if (p.id == 8):
type = 'IP'
elif (p.id == 9):
type = 'URL'
try:
if "schemas.android.com" in line:
for match in pattern.finditer(line):
if match.group():
type = ''
match_str = match.group()
if (p.id == 8):
type = 'IP'
elif (p.id == 9):
type = 'URL'
try:
if "schemas.android.com" in line:
break
url = urllib.parse.urlsplit(match_str)
if (settings.MALWARE_ENABLED):
m = Malware.objects.get(url__icontains=url.netloc)
except Exception as e:
logger.error("not found " + match_str)
elif (p.id == 10):
type = 'email'
elif (p.id == 11):
type = 'DNI'
elif (p.id == 12):
type = 'username'
elif (p.id == 13):
type = 'credentials'
elif (p.id == 14):
type = 'sensitive info'
elif (p.id == 15):
type = 'connection'
elif(p.id == 21):
try:
int(match_str, 16)
type = 'hex'
except Exception as e:
break
url = urllib.parse.urlsplit(match_str)
if (settings.MALWARE_ENABLED):
m = Malware.objects.get(url__icontains=url.netloc)
except Exception as e:
logger.error("not found " + match_str)
elif (p.id == 10):
type = 'email'
elif (p.id == 11):
type = 'DNI'
elif (p.id == 12):
type = 'username'
elif (p.id == 13):
type = 'credentials'
elif (p.id == 14):
type = 'sensitive info'
elif (p.id == 15):
type = 'connection'
elif(p.id == 21):
try:
int(match_str, 16)
type = 'hex'
except Exception as e:
break
elif (p.id == 22):
if (base64.b64encode(base64.b64decode(match_str)) != match_str):
break
type = 'base64'
finding = Finding(
scan = scan,
path = name.replace(dir, ""),
line_number = i,
line = line,
snippet = prev_line + '\n' + line + '\n' + linecache.getline(name, i + 1),
match = match_str,
status = Status.TD,
type = p,
name = p.default_name,
description = p.default_description,
severity = p.default_severity,
mitigation = p.default_mitigation,
cwe = p.default_cwe,
risk = p.default_risk,
user = scan.user
)
finding.save()
scan.findings = int(scan.findings) + 1
scan.save()
if (type != ''):
s = String(type = type, value = match_str, scan = scan, finding = finding)
s.save()
if (type == 'URL'):
if (m):
u = Domain(scan = scan, domain = url.netloc, finding = finding, malware = m)
else:
u = Domain(scan = scan, domain = url.netloc, finding = finding)
u.save()
elif (p.id == 22):
if (base64.b64encode(base64.b64decode(match_str)) != match_str):
break
type = 'base64'

match_lines = get_match_lines(line, get_position(match))
snippet = match_lines[1]
position = match_lines[0]
if (not snippet):
snippet = line

finding = Finding(
scan = scan,
path = name.replace(dir, ""),
line_number = position,
line = match_str,
snippet = snippet,
match = match_str,
status = Status.TD,
type = p,
name = p.default_name,
description = p.default_description,
severity = p.default_severity,
mitigation = p.default_mitigation,
cwe = p.default_cwe,
risk = p.default_risk,
user = scan.user
)
finding.save()
scan.findings = int(scan.findings) + 1
scan.save()
if (type != ''):
s = String(type = type, value = match_str, scan = scan, finding = finding)
s.save()
if (type == 'URL'):
if (m):
u = Domain(scan = scan, domain = url.netloc, finding = finding, malware = m)
else:
u = Domain(scan = scan, domain = url.netloc, finding = finding)
u.save()
except Exception as e:
logger.debug(e)


logger.error(e)

def get_lines(finding='', path=''):
formatter = HtmlFormatter(linenos=False, cssclass="source")
if (finding):
Expand Down
22 changes: 11 additions & 11 deletions app/templates/scan.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,9 @@ <h5 class="card-title">Permissions</h5>
<table id="permissions" class="table table-striped table-bordered table-order">
<thead>
<tr>
<th style="width: 5%;">ID</th>
<th style="width: 70%;">Name</th>
<th style="width: 15%;">Type</th>
<th style="width: 6%;">ID</th>
<th style="width: 60%;">Name</th>
<th style="width: 14%;">Type</th>
<th style="width: 10%;">Severity</th>
<th style="width: 10%;">Status</th>
</tr>
Expand Down Expand Up @@ -244,9 +244,9 @@ <h5 class="card-title">Activities</h5>
<table class="table table-striped table-bordered table-order">
<thead>
<tr>
<th style="width: 5%;">ID</th>
<th style="width: 6%;">ID</th>
<th style="width: 70%;">Name</th>
<th style="width: 15%;">Main</th>
<th style="width: 24%;">Main</th>
</tr>
</thead>
<tbody>
Expand Down Expand Up @@ -275,10 +275,10 @@ <h5 class="card-title">Components</h5>
<table class="table table-bordered table-order">
<thead>
<tr>
<th style="width: 5%;">ID</th>
<th style="width: 6%;">ID</th>
<th style="width: 10%;">Type</th>
<th style="width: 40%;">Name</th>
<th style="width: 40%;">Intents</th>
<th style="width: 44%;">Intents</th>
</tr>
</thead>
<tbody>
Expand All @@ -292,7 +292,7 @@ <h5 class="card-title">Components</h5>
<table class="table table-borderless table-striped">
<thead>
<tr>
<th style="width: 5%;">ID</th>
<th style="width: 6%;">ID</th>
<th style="width: 35%;">Intent</th>
<th style="width: 20%;">Action</th>
</tr>
Expand Down Expand Up @@ -574,10 +574,10 @@ <h5 class="card-title">Findings</h5>
<thead>
<tr>
<th style="width: 4%;"><input id="id_select_all" name="select_all" type="checkbox"/></th>
<th style="width: 5%;">ID</th>
<th style="width: 6%;">ID</th>
<th style="width: 9%;">Finding</th>
<th style="width: 7%;">Number</th>
<th style="width: 85%;">Findings</th>
<th style="width: 8%;">Number</th>
<th style="width: 75%;">Findings</th>
</tr>
</thead>
<tbody>
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ django-widget-tweaks==1.4.8
django-fontawesome-5==1.0.18
django-getenv==1.3.2
androguard==3.4.0a1
requests==2.26.0
requests==2.27.1
pdfkit==0.6.1
uwsgi==2.0.19.1
Pygments==2.10.0
Expand All @@ -15,4 +15,4 @@ Celery==5.2.2
django-filter==2.4.0
sqlalchemy==1.4.23
django-extensions==3.1.3
setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
setuptools==65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

0 comments on commit 21d4029

Please sign in to comment.