Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixing indentation #122

Merged
merged 2 commits into from
Sep 20, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions modules/getemails.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,16 @@

def get_mails(soup):
"""
Searches for <a href> tags for links then checks if link contains the
substring 'mailto' indicating that it's an email. If it is determined
to be an email then the link is split and the username is appeneded to
the list
Searches for <a href> tags for links then checks if link contains the
substring 'mailto' indicating that it's an email. If it is determined
to be an email then the link is split and the username is appeneded to
the list

Args:
soup: BeautifulSoup isntance that will be used for parsing
Args:
soup: BeautifulSoup isntance that will be used for parsing

Returns:
emails: list of email IDs
Returns:
emails: list of email IDs
"""

if isinstance(type(soup), type(BeautifulSoup)):
Expand Down
16 changes: 8 additions & 8 deletions modules/pagereader.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@

def display_url(url):
"""
Prints the status of a url based on if it can be reached using a GET
request. url is printed with a color based on status.
Green for a reachable status code and red for not reachable.

Args:
url (str): url to be printed
Returns:
None
Prints the status of a url based on if it can be reached using a GET
request. url is printed with a color based on status.
Green for a reachable status code and red for not reachable.

Args:
url (str): url to be printed
Returns:
None
"""
resp = get_url_status(url)
if resp != 0:
Expand Down
8 changes: 4 additions & 4 deletions modules/savefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

def saveJson(datatype, data):
"""
Creates json file and stores json
Creates json file and stores json

Args:
datatype: the type of the object being passed
data = data that is being stored with object
Args:
datatype: the type of the object being passed
data = data that is being stored with object
"""

timestr = time.strftime("%Y%m%d-%H%M%S")
Expand Down
5 changes: 2 additions & 3 deletions modules/updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@
def updateTor():

"""
Currently updates Tor by calling terminal commands using subprocess
Not a great method and will be replaced in the future.

Currently updates Tor by calling terminal commands using subprocess
Not a great method and will be replaced in the future.
"""

print("Checking for latest stable release")
Expand Down
108 changes: 54 additions & 54 deletions modules/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,23 @@

def bfs_urls(urls, add_exts, rec_depth=0, stop_depth=None, target_url=None):
"""
Traverses urls passed using Breadth First Search. You can specify stop
depth or specify a target to look for. The rec_depth argument is used
for recursion.

*NOTE: This function uses a GET request for each url found, this can
be very expensive so avoid if possible try to acquire the urls to
be traversed and use bfs function.

Args:
urls (list): urls to traverse
add_exts (str): additional extensions to use
rec_depth (int): used for recursion
stop_depth (int): stops traversing at this depth if specified
target_url (str): stops at this url if specified

Returns:
rec_depth (int): depth stopped at
Traverses urls passed using Breadth First Search. You can specify stop
depth or specify a target to look for. The rec_depth argument is used
for recursion.

*NOTE: This function uses a GET request for each url found, this can
be very expensive so avoid if possible try to acquire the urls to
be traversed and use bfs function.

Args:
urls (list): urls to traverse
add_exts (str): additional extensions to use
rec_depth (int): used for recursion
stop_depth (int): stops traversing at this depth if specified
target_url (str): stops at this url if specified

Returns:
rec_depth (int): depth stopped at
"""

if rec_depth == stop_depth:
Expand Down Expand Up @@ -62,18 +62,18 @@ def bfs_urls(urls, add_exts, rec_depth=0, stop_depth=None, target_url=None):

def bfs(nodes, target_node=None, rec_depth=0, stop_depth=None):
"""
Traverses nodes using Breadth First Search. You can specify stop
depth or specify a target to look for. The rec_depth argument is used
for recursion.

Args:
nodes (list): objects to traverse
target_node (object): object being searched for
rec_depth (int): used for recursion
stop_depth (int): stops traversing at this depth if specified

Returns:
rec_depth (int): depth stopped at
Traverses nodes using Breadth First Search. You can specify stop
depth or specify a target to look for. The rec_depth argument is used
for recursion.

Args:
nodes (list): objects to traverse
target_node (object): object being searched for
rec_depth (int): used for recursion
stop_depth (int): stops traversing at this depth if specified

Returns:
rec_depth (int): depth stopped at
"""

if rec_depth == stop_depth:
Expand Down Expand Up @@ -103,15 +103,15 @@ def bfs(nodes, target_node=None, rec_depth=0, stop_depth=None):

def exec_tasks(que, task_func, tasks_args=tuple()):
"""
Executes tasks inside of queue using function and arguments passed
inside of threads

Args:
que (queue.Queue): contains tasks
task_func (function): function to be executed on tasks and args
task_args (tuple): contains arguments for function
Returns:
None
Executes tasks inside of queue using function and arguments passed
inside of threads

Args:
que (queue.Queue): contains tasks
task_func (function): function to be executed on tasks and args
task_args (tuple): contains arguments for function
Returns:
None
"""
while True:
task = que.get()
Expand All @@ -124,15 +124,15 @@ def exec_tasks(que, task_func, tasks_args=tuple()):

def queue_tasks(tasks, task_func, tasks_args=tuple()):
"""
Starts threads with tasks and queue, then queues tasks and spawned
threads begin to pull tasks off queue to execute

Args:
tasks (list): lists of values that you'd like to operate on
task_func (function): function that you would like to use
tasks_args (tuple): arguments for function
Returns:
None
Starts threads with tasks and queue, then queues tasks and spawned
threads begin to pull tasks off queue to execute

Args:
tasks (list): lists of values that you'd like to operate on
task_func (function): function that you would like to use
tasks_args (tuple): arguments for function
Returns:
None
"""
que = Queue(len(tasks)*2)
for _ in tasks:
Expand All @@ -157,16 +157,16 @@ def queue_tasks(tasks, task_func, tasks_args=tuple()):

def get_url_status(url, headers=False):
"""
Uses GET request to check if website exists
Uses GET request to check if website exists

*NOTE: May look into changing this to HEAD requests to improve perf
*NOTE: May look into changing this to HEAD requests to improve perf

Args:
url (str): url to be tested
Args:
url (str): url to be tested

Return:
something? (int/Response object): return value of the connection
object's GET request if successful & zero upon failure
Return:
something? (int/Response object): return value of the connection
object's GET request if successful & zero upon failure
"""
try:
if headers:
Expand Down