Skip to content

Commit

Permalink
Move user_agent to utils
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinlul committed Nov 12, 2024
1 parent 42f7667 commit d4d4d9b
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 15 deletions.
9 changes: 2 additions & 7 deletions src/incremental.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# SPDX-FileCopyrightText: © 2022–2023 Kevin Lu
# SPDX-FileCopyrightText: © 2022–2024 Kevin Lu
# SPDX-Licence-Identifier: LGPL-3.0-or-later
import logging
from platform import python_version
import random
import sys
from time import sleep
Expand All @@ -10,11 +9,7 @@
import httpx
from ruamel.yaml import YAML

from utils import download


# User agents containing the substring "python" in any casing are forbidden after 2023-01-28
user_agent = f"https://github.com/DawnbrandBots/yaml-yugipedia httpx/{httpx.__version__} py/{python_version()}"
from utils import download, user_agent


def skip_condition(page) -> bool:
Expand Down
9 changes: 2 additions & 7 deletions src/main.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# SPDX-FileCopyrightText: © 2022–2023 Kevin Lu
# SPDX-FileCopyrightText: © 2022–2024 Kevin Lu
# SPDX-Licence-Identifier: LGPL-3.0-or-later
import logging
from platform import python_version
import random
import sys
from time import sleep
Expand All @@ -10,11 +9,7 @@
import httpx
from ruamel.yaml import YAML

from utils import download


# User agents containing the substring "python" in any casing are forbidden after 2023-01-28
user_agent = f"https://github.com/DawnbrandBots/yaml-yugipedia httpx/{httpx.__version__} py/{python_version()}"
from utils import download, user_agent


def main():
Expand Down
7 changes: 6 additions & 1 deletion src/utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: © 2022–2023 Kevin Lu
# SPDX-FileCopyrightText: © 2022–2024 Kevin Lu
# SPDX-Licence-Identifier: LGPL-3.0-or-later
from logging import getLogger
from platform import python_version
import random
from time import sleep
from typing import Optional
Expand All @@ -13,6 +14,10 @@
logger = getLogger(__name__)


# User agents containing the substring "python" in any casing are forbidden after 2023-01-28
user_agent = f"https://github.com/DawnbrandBots/yaml-yugipedia httpx/{httpx.__version__} py/{python_version()}"


def get_retry(client: httpx.Client, url: str) -> httpx.Response:
for retry in range(5):
try:
Expand Down

0 comments on commit d4d4d9b

Please sign in to comment.