From 698c44969aef3b9c5e3bec7817f930fd03a65ca1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 21 Apr 2020 17:18:34 -0700 Subject: [PATCH 1/2] Add an example unresolver Turn a URL into the component parts that our views would use to process them. This is useful for lots of places, like where we want to figure out exactly what file a URL maps to --- common | 2 +- readthedocs/core/unresolver.py | 68 +++++++++++++++ .../rtd_tests/tests/test_unresolver.py | 85 +++++++++++++++++++ readthedocs/settings/docker_compose.py | 7 +- 4 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 readthedocs/core/unresolver.py create mode 100644 readthedocs/rtd_tests/tests/test_unresolver.py diff --git a/common b/common index 5ff15ec196a..042949ff113 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 5ff15ec196a50ac538632e52eef1763ee0caa045 +Subproject commit 042949ff11321a9d044efdf41b0620089aac1981 diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py new file mode 100644 index 00000000000..3cbcf514121 --- /dev/null +++ b/readthedocs/core/unresolver.py @@ -0,0 +1,68 @@ +import logging +from urllib.parse import urlparse +from collections import namedtuple + +from django.urls import resolve as url_resolve +from django.test.client import RequestFactory + +from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.proxito.middleware import map_host_to_project_slug +from readthedocs.proxito.views.utils import _get_project_data_from_request +from readthedocs.proxito.views.mixins import ServeDocsMixin + +log = logging.getLogger(__name__) + +UnresolvedObject = namedtuple( + 'Unresolved', 'project, language_slug, version_slug, filename, fragment') + + +class UnresolverBase: + + def unresolve(self, uri): + """ + Turn a URL into the component parts that our views would use to process them. + + This is useful for lots of places, + like where we want to figure out exactly what file a URL maps to. + """ + parsed = urlparse(uri) + domain = parsed.netloc.split(':', 1)[0] + path = parsed.path + + # TODO: Make this not depend on the request object, + # but instead move all this logic here working on strings. + request = RequestFactory().get(path=path, HTTP_HOST=domain) + project_slug = request.host_project_slug = map_host_to_project_slug(request) + + # Handle returning a response + if hasattr(project_slug, 'status_code'): + return UnresolvedObject(None, None, None, None, None) + + _, __, kwargs = url_resolve( + path, + urlconf='readthedocs.proxito.urls', + ) + + mixin = ServeDocsMixin() + version_slug = mixin.get_version_from_host(request, kwargs.get('version_slug')) + + final_project, lang_slug, version_slug, filename = _get_project_data_from_request( # noqa + request, + project_slug=project_slug, + subproject_slug=kwargs.get('subproject_slug'), + lang_slug=kwargs.get('lang_slug'), + version_slug=version_slug, + filename=kwargs.get('filename', ''), + ) + + log.info('Unresolved: %s', locals()) + return UnresolvedObject(final_project, lang_slug, version_slug, filename, parsed.fragment) + + +class Unresolver(SettingsOverrideObject): + + _default_class = UnresolverBase + + +unresolver = Unresolver() +unresolve = unresolver.unresolve diff --git a/readthedocs/rtd_tests/tests/test_unresolver.py b/readthedocs/rtd_tests/tests/test_unresolver.py new file mode 100644 index 00000000000..9460e3a203b --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_unresolver.py @@ -0,0 +1,85 @@ +from django.test import override_settings +import django_dynamic_fixture as fixture +import pytest + +from readthedocs.rtd_tests.tests.test_resolver import ResolverBase +from readthedocs.core.unresolver import unresolve +from readthedocs.projects.models import Domain + + +@override_settings( + PUBLIC_DOMAIN='readthedocs.io', + RTD_EXTERNAL_VERSION_DOMAIN='dev.readthedocs.build', +) +@pytest.mark.proxito +class UnResolverTests(ResolverBase): + + def test_unresolver(self): + parts = unresolve('http://pip.readthedocs.io/en/latest/foo.html#fragment') + self.assertEqual(parts.project.slug, 'pip') + self.assertEqual(parts.language_slug, 'en') + self.assertEqual(parts.version_slug, 'latest') + self.assertEqual(parts.filename, 'foo.html') + self.assertEqual(parts.fragment, 'fragment') + + def test_unresolver_subproject(self): + parts = unresolve('http://pip.readthedocs.io/projects/sub/ja/latest/foo.html') + self.assertEqual(parts.project.slug, 'sub') + self.assertEqual(parts.language_slug, 'ja') + self.assertEqual(parts.version_slug, 'latest') + self.assertEqual(parts.filename, 'foo.html') + + def test_unresolver_translation(self): + parts = unresolve('http://pip.readthedocs.io/ja/latest/foo.html') + self.assertEqual(parts.project.slug, 'trans') + self.assertEqual(parts.language_slug, 'ja') + self.assertEqual(parts.version_slug, 'latest') + self.assertEqual(parts.filename, 'foo.html') + + def test_unresolver_domain(self): + self.domain = fixture.get( + Domain, + domain='docs.foobar.com', + project=self.pip, + canonical=True, + ) + parts = unresolve('http://docs.foobar.com/en/latest/') + self.assertEqual(parts.project.slug, 'pip') + self.assertEqual(parts.language_slug, 'en') + self.assertEqual(parts.version_slug, 'latest') + self.assertEqual(parts.filename, '') + + def test_unresolver_single_version(self): + self.pip.single_version = True + parts = unresolve('http://pip.readthedocs.io/') + self.assertEqual(parts.project.slug, 'pip') + self.assertEqual(parts.language_slug, None) + self.assertEqual(parts.version_slug, None) + self.assertEqual(parts.filename, '') + + def test_unresolver_subproject_alias(self): + relation = self.pip.subprojects.first() + relation.alias = 'sub_alias' + relation.save() + parts = unresolve('http://pip.readthedocs.io/projects/sub_alias/ja/latest/') + self.assertEqual(parts.project.slug, 'sub') + self.assertEqual(parts.language_slug, 'ja') + self.assertEqual(parts.version_slug, 'latest') + self.assertEqual(parts.filename, '') + + def test_unresolver_external_version(self): + ver = self.pip.versions.first() + ver.type = 'external' + ver.slug = '10' + parts = unresolve('http://pip--10.dev.readthedocs.build/en/10/') + self.assertEqual(parts.project.slug, 'pip') + self.assertEqual(parts.language_slug, 'en') + self.assertEqual(parts.version_slug, '10') + self.assertEqual(parts.filename, '') + + def test_unresolver_unknown_host(self): + parts = unresolve('http://random.stuff.com/en/latest/') + self.assertEqual(parts.project, None) + self.assertEqual(parts.language_slug, None) + self.assertEqual(parts.version_slug, None) + self.assertEqual(parts.filename, None) diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index 41aac9ee095..c8c61d745d1 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -38,11 +38,11 @@ class DockerBaseSettings(CommunityDevSettings): if ips and not HOSTIP: HOSTIP = ips[0][:-1] + "1" + # Turn this on to test ads + USE_PROMOS = False ADSERVER_API_BASE = f'http://{HOSTIP}:5000' - # Create a Token for an admin User and set it here. ADSERVER_API_KEY = None - ADSERVER_API_TIMEOUT = 2 # seconds - Docker for Mac is very slow # Enable auto syncing elasticsearch documents @@ -136,3 +136,6 @@ def DATABASES(self): # noqa # Remove the checks on the number of fields being submitted # This limit is mostly hit on large forms in the Django admin DATA_UPLOAD_MAX_NUMBER_FIELDS = None + + # This allows us to have CORS work well in dev + CORS_ORIGIN_ALLOW_ALL = True From df4b9cd10a5c77359f36da8b304e5ff459240f80 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 5 May 2020 06:13:57 -0700 Subject: [PATCH 2/2] Review feedback --- readthedocs/core/unresolver.py | 2 +- readthedocs/rtd_tests/tests/test_unresolver.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py index 3cbcf514121..c547092c261 100644 --- a/readthedocs/core/unresolver.py +++ b/readthedocs/core/unresolver.py @@ -36,7 +36,7 @@ def unresolve(self, uri): # Handle returning a response if hasattr(project_slug, 'status_code'): - return UnresolvedObject(None, None, None, None, None) + return None _, __, kwargs = url_resolve( path, diff --git a/readthedocs/rtd_tests/tests/test_unresolver.py b/readthedocs/rtd_tests/tests/test_unresolver.py index 9460e3a203b..cb17daa6d29 100644 --- a/readthedocs/rtd_tests/tests/test_unresolver.py +++ b/readthedocs/rtd_tests/tests/test_unresolver.py @@ -79,7 +79,4 @@ def test_unresolver_external_version(self): def test_unresolver_unknown_host(self): parts = unresolve('http://random.stuff.com/en/latest/') - self.assertEqual(parts.project, None) - self.assertEqual(parts.language_slug, None) - self.assertEqual(parts.version_slug, None) - self.assertEqual(parts.filename, None) + self.assertEqual(parts, None)