Serve non-html at documentation domain though El Proxito

dockerfiles/nginx/web.conf

@@ -18,18 +18,4 @@ server {
     location / {
         proxy_pass http://web:8000/;
     }
-
-    # Sendfile support to serve non-html files under "/dashboard/<project_slug>/downloads/<type>/<version_slug>"
-    location /proxito/ {
-        internal;
-        # Nginx will strip the `/proxito/` and pass just the `<type>/$proj/$ver/$filename`
-        # So we need to make sure we pass it to `media/` for the proper URL
-        proxy_pass http://storage:10000/;
-        proxy_set_header Host storage:10000;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Host $host;
-
-        add_header X-Served Nginx-Proxito-Sendfile always;
-    }
 }

docs/api/v3.rst

@@ -319,9 +319,9 @@ Version detail
             "type": "tag",
             "last_build": "{BUILD}",
             "downloads": {
-                "pdf": "https://readthedocs.org/projects/pip/downloads/pdf/stable/",
-                "htmlzip": "https://readthedocs.org/projects/pip/downloads/htmlzip/stable/",
-                "epub": "https://readthedocs.org/projects/pip/downloads/epub/stable/"
+                "pdf": "https://pip.readthedocs.io/_/downloads/pdf/pip/stable/",
+                "htmlzip": "https://pip.readthedocs.io/_/downloads/htmlzip/pip/stable/",
+                "epub": "https://pip.readthedocs.io/_/downloads/epub/pip/stable/"
             },
             "urls": {
                 "documentation": "https://pip.pypa.io/en/stable/",

readthedocs/api/v3/serializers.py

@@ -236,7 +236,12 @@ def get_downloads(self, obj):
         data = {}
 
         for k, v in downloads.items():
-            if k in ('htmlzip', 'pdf', 'epub'):
+            if k in ('html', 'pdf', 'epub'):
+
+                # Keep backward compatibility
+                if k == 'html':
+                    k = 'htmlzip'
+
                 data[k] = ('http:' if settings.DEBUG else 'https:') + v
 
         return data

readthedocs/api/v3/tests/mixins.py

@@ -6,6 +6,7 @@
 from django.contrib.auth.models import User
 from django.core.cache import cache
 from django.test import TestCase
+from django.test.utils import override_settings
 from django.utils.timezone import make_aware
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
@@ -16,6 +17,12 @@
 from readthedocs.redirects.models import Redirect
 
 
+@override_settings(
+    PUBLIC_DOMAIN='readthedocs.io',
+    PRODUCTION_DOMAIN='readthedocs.org',
+    USE_SUBDOMAIN=True,
+    RTD_BUILD_MEDIA_STORAGE='readthedocs.rtd_tests.storage.BuildMediaFileSystemStorageTest',
+)
 class APIEndpointMixin(TestCase):
 
     fixtures = []
@@ -97,6 +104,13 @@ def setUp(self):
             versions=[],
         )
 
+        # Make all non-html true so responses are complete
+        self.project.versions.update(
+            has_pdf=True,
+            has_epub=True,
+            has_htmlzip=True,
+        )
+
         self.client = APIClient()
 
     def tearDown(self):

readthedocs/api/v3/tests/responses/projects-detail.json

@@ -3,7 +3,11 @@
         {
             "active": true,
             "built": true,
-            "downloads": {},
+            "downloads": {
+                "epub": "https://project.readthedocs.io/_/downloads/epub/project/v1.0/",
+                "htmlzip": "https://project.readthedocs.io/_/downloads/htmlzip/project/v1.0/",
+                "pdf": "https://project.readthedocs.io/_/downloads/pdf/project/v1.0/"
+            },
             "id": 2,
             "identifier": "a1b2c3",
             "last_build": {
@@ -38,7 +42,7 @@
             "slug": "v1.0",
             "type": "tag",
             "urls": {
-                "documentation": "http://readthedocs.org/docs/project/en/v1.0/",
+                "documentation": "http://project.readthedocs.io/en/v1.0/",
                 "vcs": "https://github.com/rtfd/project/tree/v1.0/"
             },
             "verbose_name": "v1.0"
@@ -83,7 +87,7 @@
     "translation_of": null,
     "urls": {
         "builds": "https://readthedocs.org/projects/project/builds/",
-        "documentation": "http://readthedocs.org/docs/project/en/latest/",
+        "documentation": "http://project.readthedocs.io/en/latest/",
         "home": "https://readthedocs.org/projects/project/",
         "versions": "https://readthedocs.org/projects/project/versions/"
     },

readthedocs/api/v3/tests/responses/projects-list.json

@@ -28,7 +28,7 @@
             "translation_of": null,
             "urls": {
                 "builds": "https://readthedocs.org/projects/project/builds/",
-                "documentation": "http://readthedocs.org/docs/project/en/latest/",
+                "documentation": "http://project.readthedocs.io/en/latest/",
                 "home": "https://readthedocs.org/projects/project/",
                 "versions": "https://readthedocs.org/projects/project/versions/"
             },

readthedocs/api/v3/tests/responses/projects-list_POST.json

@@ -34,7 +34,7 @@
     "translation_of": null,
     "urls": {
         "builds": "https://readthedocs.org/projects/test-project/builds/",
-        "documentation": "http://readthedocs.org/docs/test-project/en/latest/",
+        "documentation": "http://test-project.readthedocs.io/en/latest/",
         "home": "https://readthedocs.org/projects/test-project/",
         "versions": "https://readthedocs.org/projects/test-project/versions/"
     },

readthedocs/api/v3/tests/responses/projects-subprojects-detail.json

@@ -39,7 +39,7 @@
         "translation_of": null,
         "urls": {
             "builds": "https://readthedocs.org/projects/subproject/builds/",
-            "documentation": "http://readthedocs.org/docs/project/projects/subproject/en/latest/",
+            "documentation": "http://project.readthedocs.io/projects/subproject/en/latest/",
             "home": "https://readthedocs.org/projects/subproject/",
             "versions": "https://readthedocs.org/projects/subproject/versions/"
         },

readthedocs/api/v3/tests/responses/projects-subprojects-list.json

@@ -44,7 +44,7 @@
                 "translation_of": null,
                 "urls": {
                     "builds": "https://readthedocs.org/projects/subproject/builds/",
-                    "documentation": "http://readthedocs.org/docs/project/projects/subproject/en/latest/",
+                    "documentation": "http://project.readthedocs.io/projects/subproject/en/latest/",
                     "home": "https://readthedocs.org/projects/subproject/",
                     "versions": "https://readthedocs.org/projects/subproject/versions/"
                 },

readthedocs/api/v3/tests/responses/projects-subprojects-list_POST.json

@@ -39,7 +39,7 @@
         "translation_of": null,
         "urls": {
             "builds": "https://readthedocs.org/projects/new-project/builds/",
-            "documentation": "http://readthedocs.org/docs/project/projects/subproject-alias/en/latest/",
+            "documentation": "http://project.readthedocs.io/projects/subproject-alias/en/latest/",
             "home": "https://readthedocs.org/projects/new-project/",
             "versions": "https://readthedocs.org/projects/new-project/versions/"
         },

readthedocs/api/v3/tests/responses/projects-superproject.json

@@ -38,7 +38,7 @@
     "translation_of": null,
     "urls": {
         "builds": "https://readthedocs.org/projects/project/builds/",
-        "documentation": "http://readthedocs.org/docs/project/en/latest/",
+        "documentation": "http://project.readthedocs.io/en/latest/",
         "home": "https://readthedocs.org/projects/project/",
         "versions": "https://readthedocs.org/projects/project/versions/"
     },

readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json

@@ -59,7 +59,7 @@
         "translation_of": null,
         "urls": {
             "builds": "https://readthedocs.org/projects/project/builds/",
-            "documentation": "http://readthedocs.org/docs/project/en/latest/",
+            "documentation": "http://project.readthedocs.io/en/latest/",
             "home": "https://readthedocs.org/projects/project/",
             "versions": "https://readthedocs.org/projects/project/versions/"
         },
@@ -73,7 +73,11 @@
     "version": {
         "active": true,
         "built": true,
-        "downloads": {},
+        "downloads": {
+            "epub": "https://project.readthedocs.io/_/downloads/epub/project/v1.0/",
+            "htmlzip": "https://project.readthedocs.io/_/downloads/htmlzip/project/v1.0/",
+            "pdf": "https://project.readthedocs.io/_/downloads/pdf/project/v1.0/"
+        },
         "id": 2,
         "identifier": "a1b2c3",
         "_links": {
@@ -85,7 +89,7 @@
         "slug": "v1.0",
         "type": "tag",
         "urls": {
-            "documentation": "http://readthedocs.org/docs/project/en/v1.0/",
+            "documentation": "http://project.readthedocs.io/en/v1.0/",
             "vcs": "https://github.com/rtfd/project/tree/v1.0/"
         },
         "verbose_name": "v1.0"

readthedocs/api/v3/tests/responses/projects-versions-detail.json

@@ -1,7 +1,11 @@
 {
     "active": true,
     "built": true,
-    "downloads": {},
+    "downloads": {
+        "epub": "https://project.readthedocs.io/_/downloads/epub/project/v1.0/",
+        "htmlzip": "https://project.readthedocs.io/_/downloads/htmlzip/project/v1.0/",
+        "pdf": "https://project.readthedocs.io/_/downloads/pdf/project/v1.0/"
+    },
     "id": 2,
     "identifier": "a1b2c3",
     "_links": {
@@ -13,7 +17,7 @@
     "slug": "v1.0",
     "type": "tag",
     "urls": {
-        "documentation": "http://readthedocs.org/docs/project/en/v1.0/",
+        "documentation": "http://project.readthedocs.io/en/v1.0/",
         "vcs": "https://github.com/rtfd/project/tree/v1.0/"
     },
     "verbose_name": "v1.0"

readthedocs/projects/models.py

@@ -52,6 +52,7 @@
 
 
 log = logging.getLogger(__name__)
+DOC_PATH_PREFIX = getattr(settings, 'DOC_PATH_PREFIX', '')
 
 
 class ProjectRelationship(models.Model):
@@ -607,23 +608,36 @@ def get_production_media_path(self, type_, version_slug, include_file=True):
             )
         return path
 
-    def get_production_media_url(self, type_, version_slug, full_path=True):
+    def get_production_media_url(self, type_, version_slug):
         """Get the URL for downloading a specific media file."""
-        try:
-            path = reverse(
-                'project_download_media',
-                kwargs={
-                    'project_slug': self.slug,
-                    'type_': type_,
-                    'version_slug': version_slug,
-                },
-            )
-        except NoReverseMatch:
-            return ''
-        if full_path:
-            path = '//{}{}'.format(settings.PRODUCTION_DOMAIN, path)
+        # Use project domain for full path --same domain as docs
+        # (project-slug.{PUBLIC_DOMAIN} or docs.project.com)
+        domain = self.subdomain()
+
+        # NOTE: we can't use ``reverse('project_download_media')`` here
+        # because this URL only exists in El Proxito and this method is
+        # accessed from Web instance
+
+        if self.is_subproject:
+            # docs.example.com/_/downloads/<alias>/<lang>/<ver>/pdf/
+            path = f'//{domain}/{DOC_PATH_PREFIX}downloads/{self.alias}/{self.language}/{version_slug}/{type_}/'
+        else:
+            # docs.example.com/_/downloads/<lang>/<ver>/pdf/
+            path = f'//{domain}/{DOC_PATH_PREFIX}downloads/{self.language}/{version_slug}/{type_}/'
+
         return path
 
+    @property
+    def is_subproject(self):
+        """Return whether or not this project is a subproject."""
+        return self.superprojects.exists()
+
+    @property
+    def alias(self):
+        """Return the alias (as subproject) for this project if it's a subproject."""
+        if self.is_subproject:
+            return self.superprojects.first().alias
+
     def subdomain(self):
         """Get project subdomain from resolver."""
         return resolve_domain(self)

readthedocs/projects/urls/public.py

@@ -31,14 +31,19 @@
         public.project_downloads,
         name='project_downloads',
     ),
+
+    # NOTE: this URL is kept here only for backward compatibility to serve
+    # non-html files from the dashboard. The ``name=`` is removed to avoid
+    # generating an invalid URL by mistake (we should manually generate it
+    # pointing to the right place: "docs.domain.org/_/downloads/")
     url(
         (
             r'^(?P<project_slug>{project_slug})/downloads/(?P<type_>[-\w]+)/'
             r'(?P<version_slug>{version_slug})/$'.format(**pattern_opts)
         ),
         public.ProjectDownloadMedia.as_view(),
-        name='project_download_media',
     ),
+
     url(
         r'^(?P<project_slug>{project_slug})/badge/$'.format(**pattern_opts),
         public.project_badge,

readthedocs/projects/views/public.py

@@ -33,6 +33,7 @@
 from readthedocs.projects.models import Project
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.proxito.views.mixins import ServeDocsMixin
+from readthedocs.proxito.views.utils import _get_project_data_from_request
 
 from .base import ProjectOnboardMixin
 from ..constants import PRIVATE
@@ -269,7 +270,9 @@ def project_downloads(request, project_slug):
 
 class ProjectDownloadMedia(ServeDocsMixin, View):
 
-    def get(self, request, project_slug, type_, version_slug):
+    same_domain_url = False
+
+    def get(self, request, project_slug=None, type_=None, version_slug=None, lang_slug=None, subproject_slug=None):
         """
         Download a specific piece of media.
 
@@ -279,11 +282,10 @@ def get(self, request, project_slug, type_, version_slug):
                      It should only care about the Version permissions,
                      not the actual Project permissions.
         """
-        version = get_object_or_404(
-            Version.objects.public(user=request.user),
-            project__slug=project_slug,
-            slug=version_slug,
-        )
+        if self.same_domain_url:
+            version = self._version_same_domain_url(request, type_, lang_slug, version_slug, subproject_slug)
+        else:
+            version = self._version_dashboard_url(request, project_slug, type_, version_slug)
 
         # Send media download to analytics - sensitive data is anonymized
         analytics_event.delay(
@@ -312,6 +314,24 @@ def get(self, request, project_slug, type_, version_slug):
             download=True,
         )
 
+    def _version_same_domain_url(self, request, type_, lang_slug, version_slug, subproject_slug=None):
+        final_project, lang_slug, version_slug, filename = _get_project_data_from_request(
+            request,
+            project_slug=None,
+            subproject_slug=subproject_slug,
+            lang_slug=lang_slug,
+            version_slug=version_slug,
+        )
+        return final_project.versions.get(slug=version_slug)
+
+    def _version_dashboard_url(self, request, project_slug, type_, version_slug):
+        version = get_object_or_404(
+            Version.objects.public(user=request.user),
+            project__slug=project_slug,
+            slug=version_slug,
+        )
+        return version
+
 
 def project_versions(request, project_slug):
     """

readthedocs/proxito/urls.py

@@ -35,6 +35,7 @@
 
 from readthedocs.constants import pattern_opts
 from readthedocs.builds.constants import EXTERNAL
+from readthedocs.projects.views.public import ProjectDownloadMedia
 from readthedocs.proxito.views.serve import (
     ServeDocs,
     ServeError404,
@@ -44,8 +45,26 @@
 from readthedocs.proxito.views.redirects import redirect_page_with_filename
 from readthedocs.proxito.views.utils import fast_404
 
+DOC_PATH_PREFIX = getattr(settings, 'DOC_PATH_PREFIX', '')
 
 urlpatterns = [
+    # Serve downloads
+    url(
+        (
+            # /_/downloads/<lang>/<ver>/<type>/
+            # /_/downloads/<alias>/<lang>/<ver>/<type>/
+            r'^{DOC_PATH_PREFIX}downloads/'
+            r'(?:(?P<subproject_slug>{project_slug})/)?'
+            r'(?P<lang_slug>{lang_slug})/'
+            r'(?P<version_slug>{version_slug})/'
+            r'(?P<type_>[-\w]+)/$'.format(
+                DOC_PATH_PREFIX=DOC_PATH_PREFIX,
+                **pattern_opts)
+        ),
+        ProjectDownloadMedia.as_view(same_domain_url=True),
+        name='project_download_media',
+    ),
+
     # Serve custom 404 pages
     url(
         r'^_proxito_404_(?P<proxito_path>.*)$',