Skip to content

Commit

Permalink
fix: use .format instead of str()
Browse files Browse the repository at this point in the history
  • Loading branch information
s-pace committed Apr 17, 2019
1 parent f959c84 commit 938fe0f
Show file tree
Hide file tree
Showing 10 changed files with 51 additions and 42 deletions.
2 changes: 1 addition & 1 deletion cli/src/commands/disable_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def run(self, args):
configs, inverted, crawler_ids = get_configs_from_website()
connector_name = args[0]

make_request("/" + str(inverted[connector_name]) + "/deactivate",
make_request('/{}/deactivate'.format(inverted[connector_name]),
"PUT")

remove_crawling_issue(connector_name)
Expand Down
2 changes: 1 addition & 1 deletion cli/src/commands/enable_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def run(self, args):
configs, inverted, crawler_ids = get_configs_from_website()
connector_name = args[0]

make_request("/" + str(inverted[connector_name]) + "/activate", "PUT")
make_request('/{}/activate'.format(inverted[connector_name]), "PUT")

send_slack_notif([{
"title": "Enable connectors",
Expand Down
2 changes: 1 addition & 1 deletion cli/src/commands/reindex_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def run(self, args):

configs, inverted, crawler_ids = get_configs_from_website()
connector_name = args[0]
make_request("/" + str(inverted[connector_name]) + "/reindex", "POST")
make_request('/{}/reindex'.format(inverted[connector_name]), "POST")

send_slack_notif([{
"title": "Manually reindexed connectors",
Expand Down
25 changes: 13 additions & 12 deletions deployer/src/algolia_internal_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def get_headers():
app_id = environ.get('APPLICATION_ID_PROD').encode()
admin_api_key = environ.get('API_KEY_PROD').encode()
auth_token = b64encode(app_id + b":" + admin_api_key).decode().replace('=',
'').replace(
'').replace(
"\n", '')

return {
Expand Down Expand Up @@ -92,7 +92,7 @@ def add_user_to_index(index_name, user_email):

# User has already access to some other indices
if right:
endpoint = get_endpoint('/application_rights/' + str(right['id']))
endpoint = get_endpoint('/application_rights/{}'.format(right['id']))
requests.patch(endpoint, json=payload, headers=headers)
print(
user_email + " is already registered on algolia dashboard (has right to other DOCSEARCH indices), analytics granted to " + index_name)
Expand Down Expand Up @@ -132,18 +132,19 @@ def remove_user_from_index(index_name, user_email):
indices.remove(index_name)

if len(indices) > 0:
requests.patch(get_endpoint('/application_rights/' + str(right['id'])),
json={
'application_right': {
'application_id': APPLICATION_ID_PROD_INTERNAL,
'user_email': user_email,
'indices': indices,
'analytics': True
}
}, headers=get_headers())
requests.patch(
get_endpoint('/application_rights/{}'.format(right['id'])),
json={
'application_right': {
'application_id': APPLICATION_ID_PROD_INTERNAL,
'user_email': user_email,
'indices': indices,
'analytics': True
}
}, headers=get_headers())
else:
requests.delete(
get_endpoint('/application_rights/' + str(right['id'])),
get_endpoint('/application_rights/{}'.format(right['id'])),
headers=get_headers())

print(user_email + " uninvite from " + index_name)
2 changes: 1 addition & 1 deletion deployer/src/fetchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def get_configs_from_repos():
txt = f.read()
config = json.loads(txt, object_pairs_hook=OrderedDict)
configs[config['index_name']] = config
print(str(len(configs)) + " docs in public and private repo")
print('{} docs in public and private repo'.format(len(configs)))

return configs

Expand Down
19 changes: 12 additions & 7 deletions deployer/src/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ def make_request(endpoint, type=None, data=None, username=None, password=None,
data=data)

if r.status_code // 100 != 2:
print("ISSUE for POST request : " + url + " with params: " + str(
data))
print(
'ISSUE for POST request : {} with params: {}'.format(url,
data))
print(r.text)
return r

Expand All @@ -77,8 +78,9 @@ def make_request(endpoint, type=None, data=None, username=None, password=None,
auth=(username, password))

if r.status_code not in success_codes:
print("ISSUE for DELETE request : " + url + " with params: " + str(
data))
print(
'ISSUE for DELETE request : {} with params: {}'.format(url,
data))
return r

if type == 'PUT':
Expand All @@ -87,8 +89,9 @@ def make_request(endpoint, type=None, data=None, username=None, password=None,
data=data)
print(r.status_code)
if r.status_code // 100 != 2:
print("ISSUE for PUT request : " + url + " with params: " + str(
data))
print(
'ISSUE for PUT request : {} with params: {}'.format(url,
data))
return r

if data != None:
Expand All @@ -100,7 +103,9 @@ def make_request(endpoint, type=None, data=None, username=None, password=None,
auth=(username, password))

if r.status_code // 100 != 2:
print("ISSUE for GET request : " + url + " with params:" + data)
print(
'ISSUE for GET request : {} with params: {}'.format(url,
data))

if json_request:
r.json()
Expand Down
8 changes: 4 additions & 4 deletions scraper/src/algolia_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ def add_records(self, records, url, from_sitemap):
color = "96" if from_sitemap else "94"

print(
"\033[" + color + "m> DocSearch: \033[0m" + url + " (\033[93m" + str(
record_count) + " records\033[0m)")
'\033[{}m> DocSearch: \033[0m{}\033[93m {} records\033[0m)'.format(
color, url, record_count))

def add_synonyms(self, synonyms):
synonyms_list = []
Expand All @@ -41,8 +41,8 @@ def add_synonyms(self, synonyms):

self.algolia_index_tmp.batch_synonyms(synonyms_list)
print(
"\033[94m> DocSearch: \033[0m" + "Synonyms" + " (\033[93m" + str(
len(synonyms_list)) + " synonyms\033[0m)")
'\033[94m> DocSearch: \033[0m Synonyms (\033[93m{} synonyms\033[0m)'.format(
len(synonyms_list)))

def commit_tmp_index(self):
"""Overwrite the real index with the temporary one"""
Expand Down
13 changes: 8 additions & 5 deletions scraper/src/config/nb_hits_updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,24 @@ def __init__(self, config_file, config_content, previous_nb_hits,

def update(self, perform_update):
if self._update_needed():
print("previous nb_hits: " + str(self.previous_nb_hits) + "\n")

print('previous nb_hits: {}\n'.format(self.previous_nb_hits))
if perform_update is None:
if sys.stdout.isatty():
perform_update = confirm('Do you want to update the nb_hits in ' + self.config_file + ' ?')
perform_update = confirm(
'Do you want to update the nb_hits in {} ?'.format(
self.config_file))
else:
perform_update = True

if perform_update:
try:
self._update_config()
print("\n[OK] " + self.config_file + " has been updated")
print(
'\n[OK] {} has been updated'.format(self.config_file))
except Exception:
print(
"\n[KO] " + "Was not able to update " + self.config_file)
'\n[KO] Was not able to update {}'.format(
self.config_file))

def _update_needed(self):
return self.previous_nb_hits is None or self.previous_nb_hits != self.new_nb_hit
Expand Down
2 changes: 1 addition & 1 deletion scraper/src/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def run_config(config):

if DocumentationSpider.NB_INDEXED > 0:
algolia_helper.commit_tmp_index()
print('Nb hits: ' + str(DocumentationSpider.NB_INDEXED))
print('Nb hits: {}'.format(DocumentationSpider.NB_INDEXED))
config.update_nb_hits_value(DocumentationSpider.NB_INDEXED)
else:
print('Crawling issue: nbHits 0 for ' + config.index_name)
Expand Down
18 changes: 9 additions & 9 deletions scraper/src/strategies/default_strategy.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ def get_records_from_response(self, response):
def _update_hierarchy_with_global_content(self, hierarchy,
current_level_int):
for index in range(0, current_level_int + 1):
if 'lvl' + str(index) in self.global_content:
hierarchy['lvl' + str(index)] = self.global_content[
'lvl' + str(index)]
if 'lvl{}'.format(index) in self.global_content:
hierarchy['lvl{}'.format(index)] = self.global_content[
'lvl{}'.format(index)]

return hierarchy

Expand Down Expand Up @@ -92,8 +92,8 @@ def get_records_from_dom(self, current_page_url=None):
anchors[current_level] = Anchor.get_anchor(node)

for index in range(current_level_int + 1, 7):
hierarchy['lvl' + str(index)] = None
anchors['lvl' + str(index)] = None
hierarchy['lvl{}'.format(index)] = None
anchors['lvl{}'.format(index)] = None
previous_hierarchy = hierarchy

if self.config.only_content_level:
Expand All @@ -111,7 +111,7 @@ def get_records_from_dom(self, current_page_url=None):
node, self.get_strip_chars(current_level, selectors))

if (
content is None or content == "") and current_level == 'content':
content is None or content == "") and current_level == 'content':
continue

hierarchy, content = self._handle_default_values(hierarchy,
Expand Down Expand Up @@ -190,7 +190,7 @@ def _get_text_content_for_level(self, node, current_level, selectors):
if 'attributes' in selectors[current_level]:
attributes = {}
for attribute_name in list(selectors[current_level][
'attributes'].keys()):
'attributes'].keys()):
matching_nodes = node.xpath(
selectors[current_level]['attributes'][attribute_name][
'selector'])
Expand All @@ -212,7 +212,7 @@ def _get_text_content_for_level(self, node, current_level, selectors):
def _get_closest_anchor(anchors):
# Getting the element anchor as the closest one
for index in list(range(6, -1, -1)):
potential_anchor = anchors['lvl' + str(index)]
potential_anchor = anchors['lvl{}'.format(index)]
if potential_anchor is None:
continue
return potential_anchor
Expand Down Expand Up @@ -324,7 +324,7 @@ def _body_contains_stop_content(self, response):

def _get_url_with_anchor(self, current_page_url, anchor):
if (
not self.config.js_render or not self.config.use_anchors) and anchor is not None:
not self.config.js_render or not self.config.use_anchors) and anchor is not None:
return current_page_url + '#' + anchor

return current_page_url

0 comments on commit 938fe0f

Please sign in to comment.