Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion backend/apps/common/geocoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,15 @@


def get_location_coordinates(query, delay=2):
"""Get location geo coordinates."""
"""Get location geo coordinates.

Args:
query (str): The location query string.
delay (int, optional): Delay in seconds before making the request.

Returns:
Location|None: The geopy Location object or None if not found.

"""
time.sleep(delay)
return Nominatim(timeout=3, user_agent=get_nest_user_agent()).geocode(query)
95 changes: 85 additions & 10 deletions backend/apps/common/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,24 @@ def get_instance(cls):
return cls._instance

def is_indexable(self, name: str):
"""Check if index is on."""
"""Check if an index is enabled for indexing.

Args:
name (str): The name of the index.

Returns:
bool: True if the index is enabled, False otherwise.

"""
return name.lower() not in self.excluded_local_index_names if IS_LOCAL_BUILD else True

def load_excluded_local_index_names(self):
"""Load excluded local index names."""
"""Load excluded local index names from settings.

Returns
IndexRegistry: The current instance of the registry.

"""
excluded_names = settings.ALGOLIA_EXCLUDED_LOCAL_INDEX_NAMES
self.excluded_local_index_names = set(
(
Expand All @@ -69,12 +82,29 @@ def load_excluded_local_index_names(self):


def is_indexable(index_name: str):
"""Determine if an index should be created based on configuration."""
"""Determine if an index should be created based on configuration.

Args:
index_name (str): The name of the index.

Returns:
bool: True if the index is indexable, False otherwise.

"""
return IndexRegistry.get_instance().is_indexable(index_name)


def register(model, **kwargs):
"""Register index if configuration allows."""
"""Register an index if configuration allows.

Args:
model (Model): The Django model to register.
**kwargs: Additional arguments for the registration.

Returns:
Callable: A wrapper function for the index class.

"""

def wrapper(index_cls):
return (
Expand All @@ -91,7 +121,15 @@ class IndexBase(AlgoliaIndex):

@staticmethod
def get_client(ip_address=None):
"""Return an instance of search client."""
"""Return an instance of the search client.

Args:
ip_address (str, optional): The IP address for the client.

Returns:
SearchClientSync: The search client instance.

"""
config = SearchConfig(
settings.ALGOLIA_APPLICATION_ID,
settings.ALGOLIA_WRITE_API_KEY,
Expand All @@ -103,7 +141,13 @@ def get_client(ip_address=None):

@staticmethod
def configure_replicas(index_name: str, replicas: dict):
"""Configure replicas."""
"""Configure replicas for an index.

Args:
index_name (str): The name of the base index.
replicas (dict): A dictionary of replica names and their ranking configurations.

"""
if not is_indexable(index_name):
return # Skip replicas configuration if base index is off.

Expand All @@ -125,7 +169,15 @@ def configure_replicas(index_name: str, replicas: dict):

@staticmethod
def _parse_synonyms_file(file_path):
"""Parse synonyms file."""
"""Parse a synonyms file and return its content.

Args:
file_path (str): The path to the synonyms file.

Returns:
list: A list of parsed synonyms or None if the file is not found.

"""
try:
with Path(file_path).open("r", encoding="utf-8") as f:
file_content = f.read()
Expand Down Expand Up @@ -163,7 +215,16 @@ def _parse_synonyms_file(file_path):

@staticmethod
def reindex_synonyms(app_name, index_name):
"""Reindex synonyms."""
"""Reindex synonyms for a specific index.

Args:
app_name (str): The name of the application.
index_name (str): The name of the index.

Returns:
int or None: The number of synonyms reindexed, or None if an error occurs.

"""
file_path = Path(f"{settings.BASE_DIR}/apps/{app_name}/index/synonyms/{index_name}.txt")

if not (synonyms := IndexBase._parse_synonyms_file(file_path)):
Expand All @@ -186,7 +247,16 @@ def reindex_synonyms(app_name, index_name):
@staticmethod
@lru_cache(maxsize=1024)
def get_total_count(index_name, search_filters=None):
"""Get total count of records in index."""
"""Get the total count of records in an index.

Args:
index_name (str): The name of the index.
search_filters (str, optional): Filters to apply to the search.

Returns:
int: The total count of records in the index.

"""
client = IndexBase.get_client()
try:
search_params = {
Expand All @@ -205,7 +275,12 @@ def get_total_count(index_name, search_filters=None):
return 0

def get_queryset(self):
"""Get queryset."""
"""Get the queryset for the index.

Returns
QuerySet: The queryset of entities to index.

"""
qs = self.get_entities()

return qs[:LOCAL_INDEX_LIMIT] if IS_LOCAL_BUILD else qs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ class Command(BaseCommand):
help = "Update OWASP Nest index replicas."

def handle(self, *_args, **_options):
"""Update replicas for Algolia indices.

Args:
*_args: Positional arguments (not used).
**_options: Keyword arguments (not used).

"""
print("\n Starting replica configuration...")
ProjectIndex.configure_replicas()
print("\n Replica have been Successfully created.")
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ class Command(BaseCommand):
help = "Update OWASP Nest index synonyms."

def handle(self, *_args, **_options):
"""Update synonyms for Algolia indices.

Args:
*_args: Positional arguments (not used).
**_options: Keyword arguments (not used).

"""
print("\nThe following models synonyms were reindexed:")
for index in (IssueIndex, ProjectIndex):
count = index.update_synonyms()
Expand Down
109 changes: 98 additions & 11 deletions backend/apps/common/management/commands/generate_sitemap.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,26 @@ class Command(BaseCommand):
}

def add_arguments(self, parser):
"""Add command-line arguments to the parser.

Args:
parser (ArgumentParser): The argument parser instance.

"""
parser.add_argument(
"--output-dir",
default=settings.STATIC_ROOT,
help="Directory where sitemap files will be saved",
)

def handle(self, *args, **options):
"""Generate sitemaps for the OWASP Nest application.

Args:
*args: Positional arguments (not used).
**options: Command-line options.

"""
output_dir = Path(options["output_dir"])
output_dir.mkdir(parents=True, exist_ok=True)

Expand All @@ -58,7 +71,12 @@ def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS(f"Successfully generated sitemaps in {output_dir}"))

def generate_project_sitemap(self, output_dir):
"""Generate sitemap for projects."""
"""Generate a sitemap for projects.

Args:
output_dir (Path): The directory to save the sitemap.

"""
routes = self.static_routes["projects"]
projects = Project.objects.all()
indexable_projects = [project for project in projects if project.is_indexable]
Expand All @@ -74,7 +92,12 @@ def generate_project_sitemap(self, output_dir):
self.save_sitemap(content, output_dir / "sitemap-project.xml")

def generate_chapter_sitemap(self, output_dir):
"""Generate sitemap for chapters."""
"""Generate a sitemap for chapters.

Args:
output_dir (Path): The directory to save the sitemap.

"""
routes = self.static_routes["chapters"]
chapters = Chapter.objects.filter(is_active=True)
indexable_chapters = [chapter for chapter in chapters if chapter.is_indexable]
Expand All @@ -90,7 +113,12 @@ def generate_chapter_sitemap(self, output_dir):
self.save_sitemap(content, output_dir / "sitemap-chapters.xml")

def generate_committee_sitemap(self, output_dir):
"""Generate sitemap for committees."""
"""Generate a sitemap for committees.

Args:
output_dir (Path): The directory to save the sitemap.

"""
routes = self.static_routes["committees"]
indexable_committees = Committee.objects.filter(is_active=True)

Expand All @@ -109,7 +137,12 @@ def generate_committee_sitemap(self, output_dir):
self.save_sitemap(content, output_dir / "sitemap-committees.xml")

def generate_user_sitemap(self, output_dir):
"""Generate sitemap for users."""
"""Generate a sitemap for users.

Args:
output_dir (Path): The directory to save the sitemap.

"""
routes = self.static_routes["users"]
users = User.objects.all()
indexable_users = [user for user in users if user.is_indexable]
Expand All @@ -125,7 +158,15 @@ def generate_user_sitemap(self, output_dir):
self.save_sitemap(content, output_dir / "sitemap-users.xml")

def generate_sitemap_content(self, routes):
"""Generate sitemap content for a set of routes."""
"""Generate the XML content for a sitemap.

Args:
routes (list): A list of route dictionaries.

Returns:
str: The XML content of the sitemap.

"""
urls = []
lastmod = datetime.now(timezone.utc).strftime("%Y-%m-%d")

Expand All @@ -141,7 +182,15 @@ def generate_sitemap_content(self, routes):
return self.create_sitemap(urls)

def generate_index_sitemap(self, sitemap_files):
"""Generate the sitemap index file."""
"""Generate the sitemap index file.

Args:
sitemap_files (list): A list of sitemap file names.

Returns:
str: The XML content of the sitemap index.

"""
sitemaps = []
lastmod = datetime.now(timezone.utc).strftime("%Y-%m-%d")

Expand All @@ -152,7 +201,15 @@ def generate_index_sitemap(self, sitemap_files):
return self.create_sitemap_index(sitemaps)

def create_url_entry(self, url_data):
"""Create a URL entry for the sitemap."""
"""Create a URL entry for the sitemap.

Args:
url_data (dict): A dictionary containing URL data.

Returns:
str: The XML entry for the URL.

"""
return (
" <url>\n"
" <loc>{loc}</loc>\n"
Expand All @@ -163,13 +220,29 @@ def create_url_entry(self, url_data):
).format(**url_data)

def create_sitemap_index_entry(self, sitemap_data):
"""Create a sitemap entry for the index."""
"""Create a sitemap index entry.

Args:
sitemap_data (dict): A dictionary containing sitemap data.

Returns:
str: The XML entry for the sitemap index.

"""
return (
" <sitemap>\n <loc>{loc}</loc>\n <lastmod>{lastmod}</lastmod>\n </sitemap>"
).format(**sitemap_data)

def create_sitemap(self, urls):
"""Create the complete sitemap XML."""
"""Create the complete sitemap XML.

Args:
urls (list): A list of URL entries.

Returns:
str: The XML content of the sitemap.

"""
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
Expand All @@ -178,7 +251,15 @@ def create_sitemap(self, urls):
)

def create_sitemap_index(self, sitemaps):
"""Create the complete sitemap index XML."""
"""Create the complete sitemap index XML.

Args:
sitemaps (list): A list of sitemap index entries.

Returns:
str: The XML content of the sitemap index.

"""
return (
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
Expand All @@ -188,6 +269,12 @@ def create_sitemap_index(self, sitemaps):

@staticmethod
def save_sitemap(content, filepath):
"""Save the sitemap content to a file."""
"""Save the sitemap content to a file.

Args:
content (str): The XML content of the sitemap.
filepath (Path): The file path to save the sitemap.

"""
with filepath.open("w", encoding="utf-8") as f:
f.write(content)
Loading