[#1185] Outgoing requests tracking

src/log_outgoing_requests/admin.py

@@ -0,0 +1,46 @@
+from django.contrib import admin
+from django.utils.translation import gettext as _
+
+from .models import OutgoingRequestsLog
+
+
+@admin.register(OutgoingRequestsLog)
+class OutgoingRequestsLogAdmin(admin.ModelAdmin):
+    fields = (
+        "url",
+        "hostname",
+        "query_params",
+        "params",
+        "status_code",
+        "method",
+        "response_ms",
+        "timestamp",
+        "req_content_type",
+        "res_content_type",
+        "trace",
+    )
+    readonly_fields = fields
+    list_display = (
+        "hostname",
+        "query_params",
+        "params",
+        "status_code",
+        "method",
+        "response_ms",
+        "timestamp",
+    )
+    list_filter = ("method", "status_code", "hostname")
+    search_fields = ("params", "query_params", "hostname")
+    date_hierarchy = "timestamp"
+    show_full_result_count = False
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def query_params(self, obj):
+        return obj.query_params
+
+    query_params.short_description = _("Query parameters")

src/log_outgoing_requests/apps.py

@@ -0,0 +1,10 @@
+from django.apps import AppConfig
+
+
+class LogOutgoingRequestsConfig(AppConfig):
+    name = "log_outgoing_requests"
+
+    def ready(self):
+        from .log_requests import install_outgoing_requests_logging
+
+        install_outgoing_requests_logging()

src/log_outgoing_requests/formatters.py

@@ -0,0 +1,23 @@
+import logging
+import textwrap
+
+
+class HttpFormatter(logging.Formatter):
+    def formatMessage(self, record):
+        result = super().formatMessage(record)
+        if record.name == "requests":
+            result += textwrap.dedent(
+                """
+                ---------------- request ----------------
+                {req.method} {req.url}
+
+                ---------------- response ----------------
+                {res.status_code} {res.reason} {res.url}
+
+            """
+            ).format(
+                req=record.req,
+                res=record.res,
+            )
+
+        return result

src/log_outgoing_requests/handlers.py

@@ -0,0 +1,35 @@
+import logging
+import traceback
+from urllib.parse import urlparse
+
+from django.conf import settings
+
+
+class DatabaseOutgoingRequestsHandler(logging.Handler):
+    def emit(self, record):
+        if settings.LOG_OUTGOING_REQUESTS_DB_SAVE:
+            from .models import OutgoingRequestsLog
+
+            trace = None
+
+            # save only the requests coming from the library requests
+            if record and record.getMessage() == "Outgoing request":
+                if record.exc_info:
+                    trace = traceback.format_exc()
+
+                parsed_url = urlparse(record.req.url)
+
+                kwargs = {
+                    "url": record.req.url,
+                    "hostname": parsed_url.hostname,
+                    "params": parsed_url.params,
+                    "status_code": record.res.status_code,
+                    "method": record.req.method,
+                    "req_content_type": record.req.headers.get("Content-Type", ""),
+                    "res_content_type": record.res.headers.get("Content-Type", ""),
+                    "timestamp": record.requested_at,
+                    "response_ms": int(record.res.elapsed.total_seconds() * 1000),
+                    "trace": trace,
+                }
+
+                OutgoingRequestsLog.objects.create(**kwargs)

src/log_outgoing_requests/log_requests.py

@@ -0,0 +1,36 @@
+import logging
+
+from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
+
+from requests import Session
+
+logger = logging.getLogger("requests")
+
+
+def hook_requests_logging(response, *args, **kwargs):
+    """
+    A hook for requests library in order to add extra data to the logs
+    """
+    extra = {"requested_at": timezone.now(), "req": response.request, "res": response}
+    logger.debug("Outgoing request", extra=extra)
+
+
+def install_outgoing_requests_logging():
+    """
+    Log all outgoing requests which are made by the library requests during a session.
+    """
+
+    if hasattr(Session, "_original_request"):
+        logger.debug(
+            "Session is already patched OR has an ``_original_request`` attribute."
+        )
+        return
+
+    Session._original_request = Session.request
+
+    def new_request(self, *args, **kwargs):
+        kwargs.setdefault("hooks", {"response": hook_requests_logging})
+        return self._original_request(*args, **kwargs)
+
+    Session.request = new_request

src/log_outgoing_requests/migrations/0001_initial.py

@@ -0,0 +1,122 @@
+# Generated by Django 3.2.15 on 2023-03-15 06:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="OutgoingRequestsLog",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "url",
+                    models.URLField(
+                        blank=True,
+                        default="",
+                        help_text="The url of the outgoing request.",
+                        verbose_name="URL",
+                    ),
+                ),
+                (
+                    "hostname",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="The hostname part of the url.",
+                        max_length=255,
+                        verbose_name="Hostname",
+                    ),
+                ),
+                (
+                    "params",
+                    models.TextField(
+                        blank=True,
+                        help_text="The parameters (if they exist).",
+                        verbose_name="Parameters",
+                    ),
+                ),
+                (
+                    "status_code",
+                    models.PositiveIntegerField(
+                        blank=True,
+                        help_text="The status code of the response.",
+                        null=True,
+                        verbose_name="Status code",
+                    ),
+                ),
+                (
+                    "method",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="The type of request method.",
+                        max_length=10,
+                        verbose_name="Method",
+                    ),
+                ),
+                (
+                    "req_content_type",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="The content type of the request.",
+                        max_length=50,
+                        verbose_name="Request content type",
+                    ),
+                ),
+                (
+                    "res_content_type",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        help_text="The content type of the response.",
+                        max_length=50,
+                        verbose_name="Response content type",
+                    ),
+                ),
+                (
+                    "response_ms",
+                    models.PositiveIntegerField(
+                        blank=True,
+                        default=0,
+                        help_text="This is the response time in ms.",
+                        verbose_name="Response in ms",
+                    ),
+                ),
+                (
+                    "timestamp",
+                    models.DateTimeField(
+                        help_text="This is the date and time the API call was made.",
+                        verbose_name="Timestamp",
+                    ),
+                ),
+                (
+                    "trace",
+                    models.TextField(
+                        blank=True,
+                        help_text="Text providing information in case of request failure.",
+                        null=True,
+                        verbose_name="Trace",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Outgoing Requests Log",
+                "verbose_name_plural": "Outgoing Requests Logs",
+            },
+        ),
+    ]

src/log_outgoing_requests/models.py

@@ -0,0 +1,83 @@
+from urllib.parse import urlparse
+
+from django.db import models
+from django.utils.functional import cached_property
+from django.utils.translation import gettext_lazy as _
+
+
+class OutgoingRequestsLog(models.Model):
+    url = models.URLField(
+        verbose_name=_("URL"),
+        blank=True,
+        default="",
+        help_text=_("The url of the outgoing request."),
+    )
+    hostname = models.CharField(
+        verbose_name=_("Hostname"),
+        max_length=255,
+        default="",
+        blank=True,
+        help_text=_("The hostname part of the url."),
+    )
+    params = models.TextField(
+        verbose_name=_("Parameters"),
+        blank=True,
+        help_text=_("The parameters (if they exist)."),
+    )
+    status_code = models.PositiveIntegerField(
+        verbose_name=_("Status code"),
+        null=True,
+        blank=True,
+        help_text=_("The status code of the response."),
+    )
+    method = models.CharField(
+        verbose_name=_("Method"),
+        max_length=10,
+        default="",
+        blank=True,
+        help_text=_("The type of request method."),
+    )
+    req_content_type = models.CharField(
+        verbose_name=_("Request content type"),
+        max_length=50,
+        default="",
+        blank=True,
+        help_text=_("The content type of the request."),
+    )
+    res_content_type = models.CharField(
+        verbose_name=_("Response content type"),
+        max_length=50,
+        default="",
+        blank=True,
+        help_text=_("The content type of the response."),
+    )
+    response_ms = models.PositiveIntegerField(
+        verbose_name=_("Response in ms"),
+        default=0,
+        blank=True,
+        help_text=_("This is the response time in ms."),
+    )
+    timestamp = models.DateTimeField(
+        verbose_name=_("Timestamp"),
+        help_text=_("This is the date and time the API call was made."),
+    )
+    trace = models.TextField(
+        verbose_name=_("Trace"),
+        blank=True,
+        null=True,
+        help_text=_("Text providing information in case of request failure."),
+    )
+
+    class Meta:
+        verbose_name = _("Outgoing Requests Log")
+        verbose_name_plural = _("Outgoing Requests Logs")
+
+    def __str__(self):
+        return ("{hostname} at {date}").format(
+            hostname=self.hostname, date=self.timestamp
+        )
+
+    @cached_property
+    def query_params(self):
+        parsed_url = urlparse(self.url)
+        return parsed_url.query